#!/usr/local/bin/perl
# 
# $Header: emdb/sysman/webapps/em/WEB-INF/perl/has/hasadm_o.pl /st_emdbsa_11.2/33 2009/07/02 22:34:19 rsamaved Exp $
#
# $Header: emdb/sysman/webapps/em/WEB-INF/perl/has/hasadm_o.pl /st_emdbsa_11.2/33 2009/07/02 22:34:19 rsamaved Exp $
#
# hasadm_o.pl
# 
# Copyright (c) 2006, 2009, Oracle and/or its affiliates. All rights reserved. 
#
# Copyright (c) 2006, 2009, Oracle and/or its affiliates. All rights reserved. 
#
#    NAME
#      hasadm_o.pl - <one-line expansion of the name>
#
#    DESCRIPTION
#      This script file provide the interfaces required by the crs ui
#       for administrering clustweware
#      where the interface invokes emcrsp the output is in xml
#      where the interface invokes crs command the output is in emagent
#       em_result format
#
#    NOTES
#     IMPORTANT !!
#     ------------
#     For details on unit testing and regression testing refer to notes
#       in scripts/has/has_metrics.pl
#
#      invoke all functions thru hasadm_main, it takes care of error handling
#
#      the functions pick the crs home from the env variable CRS_HOME
#       so env CRS_HOME needs to be set before invoking this per script
#
#    MODIFIED   (MM/DD/YY)
#    kramarat    03/22/09 - Filter by host
#    vthondep    03/18/09 - uncomenting isShared
#    kramarat    02/28/09 - performance improvement
#    kramarat    11/19/08 - getdependents for multiple resources
#    kramarat    10/09/08 - actionScript multiNode copy
#    kramarat    09/24/08 - timedout nodes list
#    kramarat    09/17/08 - Start resource after creation
#    vthondep    09/16/08 -
#    ajdsouza    09/15/08 - changed crsctl check status to support nls
#    kramarat    09/10/08 - Admin managed pools info
#    vthondep    09/08/08 -
#    kramarat    08/18/08 - servergroup to serverpool syntax change
#    kramarat    07/18/08 - performance fix
#    ajdsouza    06/30/08 -
#    nachen      06/19/08 - XbranchMerge nachen_bug-7144603 from st_emdbsa_11.1
#    nachen      06/07/08 - fix internal references security issue
#    kramarat    05/27/08 - Create Action Script
#    ajdsouza    05/15/08 -
#    kramarat    05/14/08 - Hide usage for emcrsp
#    kramarat    05/08/08 - process id
#    ajdsouza    03/25/08 -
#    kramarat    02/05/08 -
#    vthondep    01/15/08 -
#    ajdsouza    05/15/07 - incorporated proxy cli changes
#    ajdsouza    10/03/06 - added all functions
#    ajdsouza    10/03/06 - added all functions
#    kramarat    08/24/06 - Perl file for CRS Start Stop
#    kramarat    08/24/06 - Creation
#
use strict;
use warnings;

use File::Spec::Functions;
use File::Basename;
use File::Path;
use Data::Dumper;
use locale;

use has::Common;
#require "./../db/rac/rac_services_TNS_o.pl";

$Data::Dumper::Indent = 2;
$Data::Dumper::Deepcopy = 1;
$Data::Dumper::Purity = 1;
$Data::Dumper::Sortkeys = 1;

my $DEV_DEBUG;
$DEV_DEBUG=1 if $ENV{HAS_RUN_SCRIPT_DEBUG};

BEGIN
{

 use POSIX qw(locale_h);

 my $clocale='C';

 for ( qw ( LC_ALL LC_COLLATE LC_CTYPE LC_TIME LC_NUMERIC LC_MESSAGES LC_MONETARY LANG LANGUAGE ) )
 {
   $ENV{$_}=$clocale;
 }

 setlocale(LC_ALL,$clocale) or warn "WARN:hasadm_o.pl Failed to set locale to $clocale \n ";

 # temporarly setting environment
 # this will be set later on in emcrsp.bin
 # !! DO NOT CHANGE THE CLUSTER NAME HERE WHEN YOU
 # CHECK IN AS THIS WILL BREAK REGRESSION
 #
 # IF YOU ARE CREATING A HAS VIEW CREATE IT
 # WITH VIEW NAME <sa view_name>t
 # e.g. if sa view is sa3114 the has view should be sa3114t
 # the clustername is the has view should be newdb_cluster
 if ( $ENV{ADE_VIEW_ROOT} and not $ENV{HAS_USE_SHIPHOME} )
 {
  my $advrt =  $ENV{ADE_VIEW_ROOT};
  $advrt  =~ s/_ag$//;
  $advrt  = $advrt."t";

  $ENV{ORA_CRS_HOME}="$advrt/oracle";
  $ENV{CSS_CLUSTERNAME}='newdb_cluster';

  #$ENV{ORA_CRS_HOME}='/ade/kramarat_newhas/oracle';
  #$ENV{CSS_CLUSTERNAME}='kramaratnewhas';

  $ENV{CRS_HOME}="$ENV{ORA_CRS_HOME}";
  $ENV{CV_HOME}="$ENV{CRS_HOME}";
  $ENV{OCR_ROOT}="$ENV{CRS_HOME}/has_work/data.ocr";
  $ENV{OCR_LOC}="$ENV{CRS_HOME}/has_work/ocr.loc";
  $ENV{OLR_LOC}="$ENV{CRS_HOME}/has_work/olr.loc";
  $ENV{OCR_DEVELOPER_ENV}='TRUE';
  $ENV{HAS_DEVELOPMENT_ENVIRONMENT}='TRUE';
  $ENV{CV_JDKHOME}="$ENV{CRS_HOME}/jdk15";
  $ENV{ORA_ENVIRON_OPTS}='true';

  my $libs = "$ENV{CRS_HOME}/lib:$ENV{CRS_HOME}/has/lib:$ENV{CRS_HOME}/opsm/lib";

  $ENV{LD_LIBRARY_PATH}="$libs:$ENV{LD_LIBRARY_PATH}" if $ENV{LD_LIBRARY_PATH};
  $ENV{LD_LIBRARY_PATH}="$libs" unless $ENV{LD_LIBRARY_PATH};

  $ENV{PATH}="$ENV{CRS_HOME}/bin:$ENV{CRS_HOME}/has/bin:$ENV{PATH}" if $ENV{PATH};
  $ENV{PATH}="$ENV{CRS_HOME}/bin:$ENV{CRS_HOME}/has/bin" unless $ENV{PATH};

 }

}

#----------------------------------------------------------------------------
# package variable to hold execution earnigns and errors
#---------------------------------------------------------------------------
my $hasadm_xml_doctype ='<?xml version=\'1.0\' encoding=\'us-ascii\'?>
<!--    DTD to instrument Oracle Clusterware Entities in EM -->';

my $xml_header_ref;
my $xml_returncode_ref;
my $xml_node_ref;
my %xml_fn_status;

# std or xml output format
my $hasadm_output_format;



#------------------------------------------------------------------------------
#subs declared
#------------------------------------------------------------------------------
sub hasadm_xmlretcode($);
sub hasadm_xmlprinterrors();

sub hasadm_init();
sub hasadm_exitfail();
sub hasadm_cleanup();
sub hasadm_main($;@);

sub hasadm_utl_get_nodename(;$);
sub hasadm_start_nodes;
sub hasadm_stop_nodes;
sub hasadm_check_nodes;
sub hasadm_nodes;
sub hasadm_servers;
sub hasadm_server_groups;
sub hasadm_server_groups_combined;
sub hasadm_crsnodes;
sub hasadm_run_crs_command;
sub hasadm_run_crs_command_su; # for farm enabling and testing
sub hasadm_resources;
sub hasadm_resources_combined;
sub hasadm_resource_types;
sub hasadm_resourceinstances;
sub hasadm_allnode_crsstatus;
sub hasadm_allnodes_with_crsstatus;
sub hasadm_get_file_content($);
sub hasadm_restricted_servers($);
sub hasadm_internalresource_types;
sub hasadm_create_action_script;
sub hasadm_save_action_script_to_all_nodes;
sub hasadm_create_and_start;
sub hasadm_save_action_script($);
sub hasadm_get_dependents_multiple_resources;
sub hasadm_register_listener;

# name : hasadm_build_em_xml
# desc : append xml header to the xml results
#
# arg :
#   ref to hash of the entities to be covered by header
#   ref to has of the errors
# return:
#  ref to hash of the header xml
sub hasadm_build_em_xml($;$)
{

  my ( @xhref ) = @_;

  my $xres_ref = has::Common::make_element('result');
  my $xbdy_ref = has::Common::make_element('body');

  my %hdr_attrs = (crs_version => '11gr1TB' , dtd_version => '1.0' );
  $xml_header_ref = has::Common::make_element('header','',\%hdr_attrs) unless $xml_header_ref;

  hasadm_xmlretcode('Failure') unless $xml_returncode_ref;

  has::Common::append_element($xres_ref,$xml_header_ref) if $xml_header_ref;
  has::Common::append_element($xbdy_ref,$xml_node_ref) if $xml_node_ref;
  has::Common::append_element($xres_ref,$xbdy_ref);

  for my $xref ( @xhref )
  {
    next unless $xref and ref($xref) =~ /HASH/i;
    has::Common::append_element($xbdy_ref,$xref);
  }

  has::Common::append_element($xbdy_ref,$xml_returncode_ref) if $xml_returncode_ref;

  return $xres_ref;
}


#------------------------------------------------------------------------------
# FUNCTION :    hasadm_xmlretcode
#
# DESC
# return the return code element for succes/failure
#
# ARGUMENTS
#  success or failure
#
#------------------------------------------------------------------------------
sub hasadm_xmlretcode($)
{
  my ( $type ) = @_;
  my $mesg = 'Fail';
  my $code = '1';

  if ( $type =~ /Success/i )
  {
    $mesg = 'Success';
    $code = '0';
  }

  $xml_returncode_ref    = has::Common::make_element('returncode');
  my $xml_ret_code_ref   = has::Common::make_element('code',$code);
  my $xml_ret_msg_ref    = has::Common::make_element('mesg',$mesg);

  has::Common::append_element($xml_returncode_ref,$xml_ret_code_ref);
  has::Common::append_element($xml_returncode_ref,$xml_ret_msg_ref);

  return 1;

}
#------------------------------------------------------------------------------
# FUNCTION :    hasadm_xmlprinterrors
#
# DESC
# print the errors in xml
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub hasadm_xmlprinterrors()
{

  my $xmlerrref = has::Common::hasGetXmlErrorStack();
  my $hdrxmlref = hasadm_build_em_xml($xmlerrref);

  has::Common::print_xml($hasadm_xml_doctype, $hdrxmlref);

  return 1;
}

#------------------------------------------------------------------------------
# FUNCTION :    hasadm_exitfail
#
# DESC
# clean up, print errors before failure exit
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub hasadm_exitfail()
{
  # log the message to the log file
  warn("DEBUG:hasadm_o.pl:hasadm_exitfail, failed execution");

  has::Common::has_handle_error('ERROR:Failed Execution');
  hasadm_xmlretcode('Failure');

  $hasadm_output_format = 'std' unless $hasadm_output_format;

  has::Common::has_printerrors('exit_fail') if $hasadm_output_format =~ /^std$/;
  hasadm_xmlprinterrors() if $hasadm_output_format =~ /^xml$/;

  hasadm_cleanup();

  return 1;
}


#------------------------------------------------------------------------------
# FUNCTION :    hasadm_init
#
# DESC
# initialize
# Perform the initialization steps
# Prepare the directory for logging
#
# ARGUMENTS
#
#-----------------------------------------------------------------------------
sub hasadm_init()
{

 #install signal handlers for warn and die
 $SIG{'__DIE__'} = sub {  has::Common::has_handle_error( @_ ); hasadm_exitfail(); exit 1};
 $SIG{'__WARN__'} = sub { has::Common::has_handle_error( @_)};

 # initialize the library
 has::Common::hasInit();

 #initialization - prepare logging directories
 $Data::Dumper::Indent = 1;

 # Save the STDERR before redirecting it to logfile
 open(OLDERR,">&STDERR");

 # If there is an error opening a log file, redirect stderr to null
 my $devnull = File::Spec->devnull;
 open(STDERR,">$devnull");

 return 1;
}


#------------------------------------------------------------------------------
# FUNCTION :    hasadm_cleanup
#
# DESC
# clean up, print results,print errors , successful exit
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub hasadm_cleanup()
{

  # Restore STDERR
  close(STDERR);

  # Restore back the stderr fd
  open(STDERR,">&OLDERR");

  close(OLDERR);

  #%xml_errors = ();
  #%hasadm_error_stack= ();
  undef $xml_header_ref;
  undef $xml_returncode_ref;
  undef $xml_node_ref;

 # cleanup the library
 has::Common::hasCleanup();

  return 1;

}


#------------------------------------------------------------------------------
# FUNCTION :    hasadm_utl_get_nodename
#
# DESC
# return the node name for the local node
#
# ARGUMENTS
# optionsl - crsHome
# null
#------------------------------------------------------------------------------
sub hasadm_utl_get_nodename(;$)
{
  # name : hasadm_nodename_fn
  # desc : it filters our the nodename
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  array
  #
  sub hasadm_nodename_fn($$)
  {
    my ( $elref, $rsref ) = @_;

    return 1
     unless $elref and $elref->{element} and $elref->{element} =~ /^nodename$/i;

    $rsref->{nodename}=$elref->{name};

    return 1;

  }

  my ( $crsHome ) = @_;

  my %reslist;

  $reslist{nodename} = has::Common::hasGetNodeName($crsHome);

  return $reslist{nodename} if $reslist{nodename};

  warn "ERROR:hasadm_o.pl:hasadm_utl_get_nodename: Cluster home CRS_HOME needs to be passed" and return 1
   unless $crsHome and $ENV{CRS_HOME};

  $crsHome = $ENV{CRS_HOME} if $ENV{CRS_HOME} and not $crsHome;

  if ( has::Common::hasCheckForEmcrsp($crsHome) )
  {
    # if the has::Common function cannot give node name then get using emcrsp
    my %command_args = (exit_failure_list => [()]);

    has::Common::hasSetCRSEnv($crsHome);
    my $cmdresults = has::Common::runsystemcommand("emcrsp em config -e node","",\%command_args);
    has::Common::hasRestoreCRSEnv();

    my  %xmlvar =  has::Common::parse_xml($cmdresults);

    has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&hasadm_nodename_fn,\%reslist);
  }

  return $reslist{nodename} if $reslist{nodename};

  warn "ERROR:hasadm_o.pl:hasadm_utl_get_nodename Failed to get the node name" and return;

}

#------------------------------------------------------------------------------
# FUNCTION :    hasadm_start_nodes
#
# DESC
# start one or more nodes
#
# ARGUMENTS
# nodelist or null
#------------------------------------------------------------------------------
sub hasadm_start_nodes
{
  my ($nodelist) = @_;

  # log the message to the log file
  warn("DEBUG:hasadm_o.pl:hasadm_start_nodes $nodelist") if $nodelist;
  warn("DEBUG:hasadm_o.pl:hasadm_start_nodes") unless $nodelist;

  warn "ERROR:hasadm_o.pl:hasadm_start_nodes: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  $nodelist = hasadm_utl_get_nodename() unless $nodelist;

  warn "ERROR:hasadm_o.pl:hasadm_start_nodes:Failed to get node list"
   unless $nodelist;

  my @nodes = split/,/,$nodelist if $nodelist;

  for my $node ( @nodes )
  {
    my %command_args = (exit_failure_list => [()], timeout=>300,tries=>1);

    has::Common::hasSetCRSEnv($ENV{CRS_HOME});
    has::Common::runsystemcommand("crsctl start clusterware -n $node","",\%command_args)
     or warn "ERROR:hasadm_o.pl:hasadm_start_nodes Failed to execute crsctl start clusterware -n $node";
    has::Common::hasRestoreCRSEnv();

    # do the job
  }

  hasadm_check_nodes($nodelist);

  return 1;

}

#------------------------------------------------------------------------------
# FUNCTION :    hasadm_stop_nodes
#
# DESC
# stop one or more nodes
#
# ARGUMENTS
#
# nodelist
#------------------------------------------------------------------------------
sub hasadm_stop_nodes
{

  my ($nodelist) = @_;

  # log the message to the log file
  warn("DEBUG:hasadm_o.pl:hasadm_stop_nodes $nodelist") if $nodelist;
  warn("DEBUG:hasadm_o.pl:hasadm_start_nodes") unless $nodelist;

  warn "ERROR:hasadm_o.pl:hasadm_stop_nodes: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  $nodelist = hasadm_utl_get_nodename() unless $nodelist;

  warn "ERROR:hasadm_o.pl:hasadm_stop_nodes:Failed to get node list"
   unless $nodelist;

  my @nodes = split/,/,$nodelist;

  for my $node ( @nodes )
  {
    my %command_args = (exit_failure_list => [()], timeout=>300,tries=>1);

    # do the job
    has::Common::hasSetCRSEnv($ENV{CRS_HOME});
    has::Common::runsystemcommand("crsctl stop clusterware -n $node","",\%command_args);
    has::Common::hasRestoreCRSEnv();

    warn "ERROR:hasadm_o.pl:hasadm_stop_nodes Failed to execute crsctl stop clusterware -n $node"
     if $command_args{command_return_status};

  }

  hasadm_check_nodes($nodelist);

  return 1;

}

#------------------------------------------------------------------------------
# FUNCTION :    hasadm_check_nodes
#
# DESC
# check crs status on one or more nodes
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
#Cluster Synchronization Services appears healthy
#Cluster Ready Services appears healthy
#Event Manager appears healthy
my %hasadm_crs_status;
$hasadm_crs_status{desc}{crsd}='Cluster Ready Services';
$hasadm_crs_status{desc}{css}='Cluster Synchronization Service';
$hasadm_crs_status{desc}{evm}='Event Manager';
$hasadm_crs_status{desc}{has}='Oracle High Availability Services';

$hasadm_crs_status{name}{crsd}='CRS-2194|CRS-4537|Cluster Ready Services|crsd';
$hasadm_crs_status{name}{css}='CRS-2186|CRS-4529|Cluster Synchronization Service|css';
$hasadm_crs_status{name}{evm}='CRS-2190|CRS-4533|Event Manager|evm';
$hasadm_crs_status{name}{has}='CRS-4638|Oracle High Availability Services';

$hasadm_crs_status{crsd}{up}='CRS-2194|CRS-4537|healthy';
$hasadm_crs_status{css}{up}='CRS-2186|CRS-4529|healthy';
$hasadm_crs_status{evm}{up}='CRS-2190|CRS-4533|healthy';
$hasadm_crs_status{has}{up}='CRS-4638|healthy';


$hasadm_crs_status{crsd}{down}='CRS-4538|CRS-4000|failure|down|cannot communicate';
$hasadm_crs_status{css}{down}='CRS-4530|CRS-4000|failure|down|cannot communicate';
$hasadm_crs_status{evm}{down}='CRS-4534|CRS-4000|failure|down|cannot communicate';
$hasadm_crs_status{has}{down}='CRS-4639|CRS-4000|failure|down|cannot communicate';


$hasadm_crs_status{command_failure}='CRS-4000|Command Check failed';


sub hasadm_check_nodes
{
  my ($nodelist) = @_;

  # log the message to the log file
  warn("DEBUG:hasadm_o.pl:hasadm_check_nodes $nodelist") if $nodelist;

  warn "ERROR:hasadm_o.pl:hasadm_check_nodes: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  my %command_args;
  my $cmd_result;

  # get the coma seperated list of crs nodes
  my $allnodes;

  if ( $nodelist )
  {
    $allnodes = $nodelist;
  }
  else
  {
   $allnodes = has::Common::hasGetNodeList($ENV{CRS_HOME});
  }

  chomp($allnodes) if $allnodes;
  my @allnodeList = split/,/,$allnodes if $allnodes;
  my %allNodeMap;
  if ( @allnodeList )
  {
      %allNodeMap = map { $_ => 1} @allnodeList;
  }

  # if a nodelist is passed use crsctl check clusterware -n node
  if ( $nodelist )
  {
    %command_args = (exit_failure_list => [()], timeout=>300);

    has::Common::hasSetCRSEnv($ENV{CRS_HOME});
    $cmd_result = has::Common::runsystemcommand("crsctl check clusterware -n $nodelist","",\%command_args);
    has::Common::hasRestoreCRSEnv();

  }
  else
  {
    %command_args = (exit_failure_list => [()]);

    has::Common::hasSetCRSEnv($ENV{CRS_HOME});
    $cmd_result = has::Common::runsystemcommand("crsctl check clusterware -all","",\%command_args);
    has::Common::hasRestoreCRSEnv();
  }


  if ( $command_args{command_return_status} and $cmd_result and $cmd_result =~ /Parse error/i )
  {
    # if there is a parse error give em_error and return
    warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed Parse error with command crsctl check clusterware";
  }
  elsif ( $cmd_result and $cmd_result =~ /Unknown Node/i )
  {
    # if there is a node unknown error give em_error and return
    warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed, Unknown Nodes from command crsctl check clusterware";
  }
  elsif ( $cmd_result and $cmd_result =~ /$hasadm_crs_status{command_failure}/i )
  {
    # if there is a node unknown error give em_error and return
    warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed, command crsctl check clusterware";
  }
  elsif ( $command_args{command_return_status} )
  {
    # if there is a execution failure give em_error and return
    warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed to execute command crsctl check clusterware";
  }
  elsif ( $cmd_result and $cmd_result =~ /timed out/i )
  {
    # if there is a timeout error give em_error and return
    warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed, Timed Out command crsctl check clusterware";
  }
  else
  {
    #crsctl check clusterware command is successful
    my  @cmd_result = split/\n/,$cmd_result if $cmd_result;

    warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed No output from command crsctl check crs"
      and return 1 unless @cmd_result;

    my $current_node;
    for my $row ( @cmd_result )
    {
      # do not proceed if blank row
      next unless $row;

      # do not proceed if row contains only asterisks
      my $valid_row = $row;
      $valid_row =~ s/\*//g;
      $valid_row =~ s/^\s+|\s+$//g;
      next unless $valid_row;

      # it is a node name row if it does not have any daemon names
      $current_node = $row
        unless $row =~ /$hasadm_crs_status{name}{crsd}/i
        or $row =~ /$hasadm_crs_status{name}{css}/i
        or $row =~ /$hasadm_crs_status{name}{evm}/i
        or $row =~ /$hasadm_crs_status{name}{has}/i;

      # remove the last colon that comes with node name
      $current_node =~ s/:*$// if $current_node;
      $current_node =~ s/^\s+|\s+$// if $current_node;

      # do not continue if there is no node name
      next unless $current_node;
      next unless exists $allNodeMap{$current_node};

      # analyze and print the status of each daemon
      for my $process ( keys %{$hasadm_crs_status{name}} )
      {
        next unless $row =~ /$hasadm_crs_status{name}{$process}/i;

        print "em_result=$current_node|$hasadm_crs_status{desc}{$process}|$process|status|up\n"
          and last if $row =~ /$hasadm_crs_status{$process}{up}/i;

        print "em_result=$current_node|$hasadm_crs_status{desc}{$process}|$process|status|down\n"
          and last;
      }

    }
    return 1;
  }

  # crsctl check clusterware fails, so use crsctl check crs for local node
  my $node = hasadm_utl_get_nodename();
  warn "ERROR:hasadm_o.pl:hasadm_check_nodes:Failed to get local node name"
   unless $node;

  %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  $cmd_result = has::Common::runsystemcommand("crsctl check crs","",\%command_args);
  has::Common::hasRestoreCRSEnv();

  # if there is a parse error give em_error and return
  warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed Parse error with command crsctl check crs"
    and return 1
   if $command_args{command_return_status}
     and $cmd_result
     and $cmd_result =~ /Parse error/i;

  warn "WARN:hasadm_o.pl:hasadm_crs_status Failed to execute crsctl check crs"
   if $command_args{command_return_status};

  # if there is a command failure return
  warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed command crsctl check crs"
    and return 1
   if $cmd_result and $cmd_result =~ /$hasadm_crs_status{command_failure}/i;

  my  @cmd_result = split/\n/,$cmd_result if $cmd_result;

  warn "ERROR:hasadm_o.pl:hasadm_crs_status Failed No output from command crsctl check crs"
  and return 1
  unless @cmd_result;

  for my $row ( @cmd_result )
  {
   next unless $row;
   for my $process ( keys %{$hasadm_crs_status{name}} )
   {
     next unless $row =~ /$hasadm_crs_status{name}{$process}/i;

     print "em_result=$node|$hasadm_crs_status{desc}{$process}|$process|status|up\n"
      and last if $row =~ /$hasadm_crs_status{$process}{up}/i;

     print "em_result=$node|$hasadm_crs_status{desc}{$process}|$process|status|down\n"
      and last;

   }

  }

  #graceful exit
  return 1;

}




#------------------------------------------------------------------------------
# FUNCTION :    hasadm_nodes
#
# DESC
# print the list of nodes on the selected cluster (just olsnodes)
#
# ARGUMENTS
#  NULL
#------------------------------------------------------------------------------
sub hasadm_nodes
{

  # get the coma seperated list of crs nodes
  my $result = has::Common::hasGetNodeList($ENV{CRS_HOME});

  chomp($result) if $result;

  $result = hasadm_utl_get_nodename() unless $result;

  chomp($result) if $result;

  print "em_result=$result\n" if $result;
  print "em_result=\n" unless $result;

  return 1;

}


# name : hasadm_resources_combined
# desc : fn to prepare and print the resouce list along with instance details from the xml results variable
#
# arg  :
#  arguments to emcrsp binary
#  arg to indicate no printing resource information only cache
# return:
#
sub hasadm_resources_combined
{
  #hasadm_main('hasadm_resources',@_);
  my ( $args,$onlycache ) = @_;

  my $cmd = 'emcrsp em config -e resource';
  $cmd .= " -n $args" if $args;

  hasadm_run_crs_command($cmd);
  print "ORACLE_EM_MULTI_COMMAND_SEPARATOR";
  #hasadm_main('hasadm_resourceinstances');
  hasadm_run_crs_command('emcrsp em status -e resource');
}

# name : hasadm_resources
# desc : fn to prepare and print the resouce list from the xml results variable
#
# arg  :
#  arguments to emcrsp binary
#  arg to indicate no printing resource information only cache
# return:
#
sub hasadm_resources
{

  # name : hasadm_resouce_fn
  # desc : for each element passed it filters our the resource elements
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  array
  #


  sub hasadm_resource_fn($$)
  {

    my ( $elref, $rsref ) = @_;

    return 1 unless $elref->{element}
       and $elref->{element} =~ /^(entity|nodename|returncode|header)$/i;

    $xml_header_ref = $elref if $elref->{element} =~ /^header$/i;
    $xml_returncode_ref = $elref if $elref->{element} =~ /^returncode$/i;
    $xml_node_ref = $elref if $elref->{element} =~ /^nodename$/i;

    return 1
     unless $elref->{attrs} and $elref->{attrs}{entity_type} and
      $elref->{attrs}{entity_type} =~ /^resource$/i;

    has::Common::append_element($rsref,$elref);

    # keep an has stack of resources
    return 1 unless $elref->{attrs}{entity_name};
    $has::Common::hasadm_resource_list{$elref->{attrs}{entity_name}}=$elref;

    return 1;

  }

  my ( $args,$onlycache ) = @_;

  # get the resource type information to be cached
  hasadm_resource_types('','onlycache');

  my %reslist;
  %has::Common::hasadm_resource_list = ();

  my $cmd = 'emcrsp em config -e resource';
  $cmd .= " -s $args" if $args;

  warn "ERROR:hasadm_o.pl:hasadm_resources: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  my %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $cmdresults = has::Common::runsystemcommand($cmd,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  my  %xmlvar =  has::Common::parse_xml($cmdresults);

  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&hasadm_resource_fn,\%reslist);

  # get the resource type chain for each resource
  for my $resname ( keys %has::Common::hasadm_resource_list )
  {
    my $resref = $has::Common::hasadm_resource_list{$resname};
    my $restype = '';
    $restype = $resref->{attrs}{type} if $resref->{attrs}{type};

    $resref->{attrs}{root_type} = '';
    $resref->{attrs}{base_type_chain} = '';

    $resref->{attrs}{root_type}=$has::Common::hasadm_restype_base{$restype}{base_type}
      if $has::Common::hasadm_restype_base{$restype}{base_type};

    $resref->{attrs}{base_type_chain}=$has::Common::hasadm_restype_base{$restype}{base_type_chain}
      if $has::Common::hasadm_restype_base{$restype}{base_type_chain};
  }

  return 1 if $onlycache;

  $reslist{element} = 'entities';

  my $xmlerrref = has::Common::hasGetXmlErrorStack();

  my  $hdrxmlref = hasadm_build_em_xml(\%reslist,$xmlerrref);

  has::Common::print_xml($hasadm_xml_doctype, $hdrxmlref);

  return 1;

}


# name : hasadm_server_groups_combined
# desc : fn to prepare and print the server group list from the xml output and
# collect CRS Admin information
#
# arg  :
#  arguments to emcrsp binary
# return:
#
sub hasadm_server_groups_combined
{
  hasadm_main('hasadm_server_groups');
  print "ORACLE_EM_MULTI_COMMAND_SEPARATOR";

  hasadm_main('hasadm_run_crs_command','crsctl query crs administrator');
  print "ORACLE_EM_MULTI_COMMAND_SEPARATOR";

  hasadm_main('hasadm_run_crs_command','srvctl config database -S 1');

  return 1;

}

# name : hasadm_server_groups
# desc : fn to prepare and print the server group list from the xml output
#
# arg  :
#  arguments to emcrsp binary
# return:
#
sub hasadm_server_groups
{
  # name : hasadm_server_group_fn
  # desc : for each element passed it filters our the server_group lements
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  array
  #
  sub hasadm_server_group_fn($$)
  {
    my ( $elref, $rsref ) = @_;

    return 1 unless $elref->{element}
     and $elref->{element} =~ /^(entity|nodename|returncode|header)$/i;

    $xml_header_ref = $elref if $elref->{element} =~ /^header$/i;
    $xml_returncode_ref = $elref if $elref->{element} =~ /^returncode$/i;
    $xml_node_ref = $elref if $elref->{element} =~ /^nodename$/i;

    return 1
     unless $elref->{attrs} and $elref->{attrs}{entity_type} and
      $elref->{attrs}{entity_type} =~ /^server_pool$/i;

    has::Common::append_element($rsref,$elref);

    return 1;

  }



  my ( $args ) = @_;

  my %reslist;

  my $cmd = 'emcrsp em config -e server_pool';
  $cmd .= " -name $args" if $args;

  warn "ERROR:hasadm_o.pl:hasadm_server_groups: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  my %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $cmdresults = has::Common::runsystemcommand($cmd,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  my  %xmlvar =  has::Common::parse_xml($cmdresults);

  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&hasadm_server_group_fn,\%reslist);

  $reslist{element} = 'entities';

  my $xmlerrref = has::Common::hasGetXmlErrorStack();

  my  $hdrxmlref = hasadm_build_em_xml(\%reslist,$xmlerrref);

  has::Common::print_xml($hasadm_xml_doctype,$hdrxmlref);


  return 1;

}

# name : hasadm_run_crs_command
# desc : fn to prepare and run crsctl command
#
# arg  :
#  full command that needs to be executed
#  eg: crsctl delete servergroup sg1
# return:
#
sub hasadm_run_crs_command
{
  my ( $cmd,$resref ) = @_;

  my %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $cmdresults = has::Common::runsystemcommand($cmd,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  print "Error executing command $cmd\n" if $command_args{command_return_status};

  #strip beginning and trailing spaces
  $cmdresults =~ s/^\s+|\s+$//g if $cmdresults;

  #replace new line characters with space if any
  $cmdresults =~ s/\n+/ /g if $cmdresults;
  #$cmdresults = "$cmdresults\n" if $cmdresults and $cmdresults !~ /\n$/;

  if ( $resref )
  {
    $resref->{cmd_results}=$cmdresults if $cmdresults;
    $resref->{command_return_status}=$command_args{command_return_status} if $command_args{command_return_status};
    return 1;
  }

  print $cmdresults;

  return 1;
}


# name : hasadm_run_crs_command_su
# desc : fn to prepare and run crsctl command as superuser
#        This is purely for testing purpose and running in farm
#        This will be involked on setting a hidden emoms property
#
# arg  :
#  full command that needs to be executed
#  eg: crsctl delete servergroup sg1
# return:
#
sub hasadm_run_crs_command_su
{
  my ( $username, $password, $cmd, $resref ) = @_;

  if ($username =~ /^aime/)
  {
    $cmd = "/usr/local/packages/aime/em/run_as_root '" . $cmd . "'";
  }
  else
  {
    # actual command
    $cmd = "echo $password | sudo -S " . $cmd;

    # execute sudo -k to clear sudo priviliges
    $cmd = $cmd . ";sudo -k";
  }

  if ( $resref )
  {
   return hasadm_run_crs_command($cmd,$resref);
  }
  else
  {
   return hasadm_run_crs_command($cmd);
  }
}

# name : hasadm_servers
# desc : fn to prepare and print the server list from the xml output
#
# arg  :
#  arguments to emcrsp binary
# return:
#
sub hasadm_servers
{
  # name : hasadm_server_fn
  # desc : for each element passed it filters our the server elements
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  array
  #
  sub hasadm_server_fn($$)
  {
    my ( $elref, $rsref ) = @_;

    return 1 unless $elref->{element}
     and $elref->{element} =~ /^(entity|nodename|returncode|header)$/i;

    $xml_header_ref = $elref if $elref->{element} =~ /^header$/i;
    $xml_returncode_ref = $elref if $elref->{element} =~ /^returncode$/i;
    $xml_node_ref = $elref if $elref->{element} =~ /^nodename$/i;

    return 1
     unless $elref->{attrs} and $elref->{attrs}{entity_type} and
      $elref->{attrs}{entity_type} =~ /^server$/i;

    has::Common::append_element($rsref,$elref);

    return 1;

  }



  my ( $args ) = @_;

  my %reslist;

  my $cmd = 'emcrsp em config -e server';
  $cmd .= " -name $args" if $args;

  warn "ERROR:hasadm_o.pl:hasadm_servers: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  my %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $cmdresults = has::Common::runsystemcommand($cmd,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  my  %xmlvar =  has::Common::parse_xml($cmdresults);

  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&hasadm_server_fn,\%reslist);

  $reslist{element} = 'entities';

  my $xmlerrref = has::Common::hasGetXmlErrorStack();

  my  $hdrxmlref = hasadm_build_em_xml(\%reslist,$xmlerrref);

  has::Common::print_xml($hasadm_xml_doctype, $hdrxmlref);

  return 1;

}


# name : hasadm_crsnodes
# desc : fn to prepare and print the node list from the xml output
#
# arg  :
#  arguments to emcrsp binary
# return:
#
sub hasadm_crsnodes
{
  # name : hasadm_crsnodes_fn
  # desc : for each element passed it filters our the node elements
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  array
  #
  sub hasadm_crsnodes_fn($$)
  {
    my ( $elref, $rsref ) = @_;

    return 1 unless $elref->{element}
     and $elref->{element} =~ /^(entity|nodename|returncode|header)$/i;

    $xml_header_ref = $elref if $elref->{element} =~ /^header$/i;
    $xml_returncode_ref = $elref if $elref->{element} =~ /^returncode$/i;
    $xml_node_ref = $elref if $elref->{element} =~ /^nodename$/i;

    return 1
     unless $elref->{attrs} and $elref->{attrs}{entity_type} and
      $elref->{attrs}{entity_type} =~ /^node$/i;

    has::Common::append_element($rsref,$elref);

    return 1;

  }



  my ( $args ) = @_;

  my %reslist;

  my $cmd = 'emcrsp em config -e node';

  $cmd .= " -name $args" if $args;

  warn "ERROR:hasadm_o.pl:hasadm_crs_nodes: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  my %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $cmdresults = has::Common::runsystemcommand($cmd,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  my  %xmlvar =  has::Common::parse_xml($cmdresults);

  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&hasadm_crsnodes_fn,\%reslist);

  $reslist{element} = 'entities';

  my $xmlerrref = has::Common::hasGetXmlErrorStack();

  my  $hdrxmlref = hasadm_build_em_xml(\%reslist,$xmlerrref);

  has::Common::print_xml($hasadm_xml_doctype, $hdrxmlref);

  return 1;

}


# name : hasadm_resource_types
# desc : fn to prepare and print the resource type list from the xml output
#
# arg  :
#  args to emcrsp for resource types
#  flag to indicate if the function is called only for caching resource type values

# return:
#
sub hasadm_resource_types
{
  # name : hasadm_resource_type_fn
  # desc : for each element passed it filters our the resource type elements
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  array
  #
  sub hasadm_resource_type_fn($$)
  {
    my ( $elref, $rsref ) = @_;

    return 1 unless $elref->{element}
      and $elref->{element} =~ /^(entity|nodename|returncode|header)$/i;

    $xml_header_ref = $elref if $elref->{element} =~ /^header$/i;
    $xml_returncode_ref = $elref if $elref->{element} =~ /^returncode$/i;
    $xml_node_ref = $elref if $elref->{element} =~ /^nodename$/i;

    return 1 unless $elref->{attrs} and $elref->{attrs}{entity_type} and $elref->{attrs}{entity_type} =~ /^resource_type$/i;

    has::Common::append_element($rsref,$elref);

    return 1;

  }



  my ( $args,$onlycache ) = @_;

  my %reslist;

  my $cmd = 'emcrsp em config -e resource_type -b all';
  $cmd .= " -name $args" if $args;

  warn "ERROR:hasadm_o.pl:hasadm_resource_types: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  my %command_args = (exit_failure_list => [()]);

    has::Common::hasSetCRSEnv($ENV{CRS_HOME});
    my $cmdresults = has::Common::runsystemcommand($cmd,"",\%command_args);
    has::Common::hasRestoreCRSEnv();

  my  %xmlvar =  has::Common::parse_xml($cmdresults);

  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&hasadm_resource_type_fn,\%reslist);

  # execute function to mark base types and base type for attribs etc
  has::Common::hasadm_resource_types_mark_base_types(\%reslist);

  return 1 if $onlycache;

  $reslist{element} = 'entities';

  my $xmlerrref = has::Common::hasGetXmlErrorStack();

  my  $hdrxmlref = hasadm_build_em_xml(\%reslist,$xmlerrref);

  has::Common::print_xml($hasadm_xml_doctype, $hdrxmlref);

  return 1;

}


# name : hasadm_resourceinstances
# desc : fn to prepare and print the resource instance list from the xml output
#
# arg  :
#  arguments to emcrsp binary
#  arg to indicate only cache no print of resource instance information
# return:
#
sub hasadm_resourceinstances
{
  # name : hasadm_resourceinstance_fn
  # desc : for each element passed it filters our the resource instance elements
  #
  # arg  :
  #  ref to xml element to be filtered
  #  ref to the result hass entities  array
  #
  sub hasadm_resourceinstance_fn($$)
  {
    my ( $elref, $rsref ) = @_;

    return 1 unless $elref->{element}
     and $elref->{element} =~ /^(entity|nodename|returncode|header|attribute)$/i;

    $xml_header_ref = $elref if $elref->{element} =~ /^header$/i;
    $xml_returncode_ref = $elref if $elref->{element} =~ /^returncode$/i;
    $xml_node_ref = $elref if $elref->{element} =~ /^nodename$/i;

    # get the resource name for a resource_instance
    if ( $elref->{element} =~ /^attribute$/i )
    {

       # attribute should have the name ,value .. child elements
       return 1 unless $elref->{children};

       my $entity_ref;

       #get the  resource_type entity
       $entity_ref = $elref->{parent}->{parent}
        if $elref->{parent}
         and $elref->{parent}->{parent};

       return 1 unless $entity_ref
        and $entity_ref->{attrs}
         and $entity_ref->{attrs}{entity_name};

       # get the default_value for name=NAME
       my $val = '';
       my $name;
       for my $elem ( @{$elref->{children}} )
       {

        next unless $elem
         and $elem->{element}
          and $elem->{element} =~ /^(name|value)$/i;

        $name = $elem->{name} and next
         if  $elem->{element} =~ /name/
           and $elem->{name};

        $val = $elem->{name} and next
         if  $elem->{element} =~ /value/
           and $elem->{name};

       }

       return 1 unless $name and $name =~ /^NAME$/i;

       $entity_ref->{attrs}{resource_name}= $val;

       return 1;
    }

    return 1
     unless $elref->{attrs} and $elref->{attrs}{entity_type} and
      $elref->{attrs}{entity_type} =~ /^resource$/i;

    has::Common::append_element($rsref,$elref);

    # keep an has stack of resource instances
    return 1 unless $elref->{attrs}{entity_name};
    $has::Common::hasadm_resourceinst_list{$elref->{attrs}{entity_name}}=$elref;

    return 1;

  }




  my ( $args, $onlycache ) = @_;

  my %reslist;
  %has::Common::hasadm_resourceinst_list = ();

  warn "ERROR:hasadm_o.pl:hasadm_resourceinstances: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  # get the resource type information to be cached
  hasadm_resources('','onlycache');

  my $cmd = 'emcrsp em status -e resource';
  $cmd .= " -name $args" if $args;

  my %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $cmdresults = has::Common::runsystemcommand($cmd,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  my  %xmlvar =  has::Common::parse_xml($cmdresults);

  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&hasadm_resourceinstance_fn,\%reslist);

  # get the resource information for each resource instance
  for my $instid ( keys %has::Common::hasadm_resourceinst_list )
  {

    my $resinref = $has::Common::hasadm_resourceinst_list{$instid};
    my $resname = '';

    $resinref->{attrs}{resource_type} = '';
    $resinref->{attrs}{root_type} = '';
    $resinref->{attrs}{base_type_chain} = '';

    $resname = $resinref->{attrs}{resource_name} if $resinref->{attrs}{resource_name};
     next unless $resname;
    my $resref;
    $resref = $has::Common::hasadm_resource_list{$resname} if $has::Common::hasadm_resource_list{$resname};
     next unless $resref;

    $resinref->{attrs}{resource_type}=$resref->{attrs}{type} if $resref->{attrs}{type};
    $resinref->{attrs}{root_type}=$resref->{attrs}{root_type} if $resref->{attrs}{root_type};
    $resinref->{attrs}{base_type_chain}=$resref->{attrs}{base_type_chain} if $resref->{attrs}{base_type_chain};

  }

  return 1 if $onlycache;

  $reslist{element} = 'entities';

  my $xmlerrref = has::Common::hasGetXmlErrorStack();

  my  $hdrxmlref = hasadm_build_em_xml(\%reslist,$xmlerrref);

  has::Common::print_xml($hasadm_xml_doctype, $hdrxmlref);

  return 1;

}


# name : hasadm_allnodes_with_crsstatus
# desc : fn to return the list of crs servers with crs status on each node
#
# arg  :
#  null
# return:
#
sub hasadm_allnodes_with_crsstatus
{
  hasadm_allnode_crsstatus();
  print "ORACLE_EM_MULTI_COMMAND_SEPARATOR";
  hasadm_nodes();
}

# name : hasadm_allnode_crsstatus
# desc : fn to return the list of crs servers with crs status on each node
#
# arg  :
#  list of nodes or null
# return:
#
sub hasadm_allnode_crsstatus
{
  my ( @args ) = @_;

  # log the message to the log file
  warn("DEBUG:hasadm_o.pl:hasadm_allnode_crsstatus in hasadm_allnode_crsstatus");

  warn "ERROR:hasadm_o.pl:hasadm_allnode_crsstatus: Cluster home CRS_HOME needs to be passed" and return 1
   unless $ENV{CRS_HOME};

  # get the coma seperated list of crs nodes
  # my $result = has::Common::hasGetNodeList($ENV{CRS_HOME});

  # $result =~ s/\s*\n\s*/,/g if $result;

  # warn "ERROR:hasadm_o.pl:hasadm_allnode_crsstatus:Failed to get list of nodes" unless $result;

  # invoke without any args.
  # doing so will invoke crsctl check clusterware -all command
  hasadm_check_nodes();

  return 1;

}

# name : hasadm_crs_restricted_servers
# desc : fn to return the list of restricted crs servers where a resource can be relocated
#
# arg  :
#  list of resources
# return:
#
sub hasadm_crs_restricted_servers($)
{
  my ($resourcelist) = @_;
  my @results;

  #initalization

  $resourcelist = 'resource1' unless $resourcelist;

  # log the message to the log file
  warn("DEBUG:hasadm_o.pl:hasadm_crs_restricted_servers $resourcelist");

  my @resources = split/,/,$resourcelist;

  for my $resource ( @resources )
  {
    # do the job

    push @results,"em_result=$resource|node1";
    push @results,"em_result=$resource|node3";

  }

print "<?xml version='1.0' encoding='us-ascii'?> \n";
  print "<result> \n";
print "<header crs_version='11.0' dtd_version='1.0'></header> \n";

print "<body> \n";
print "<entities> \n";
print "<entity entity_name='stbdq16' entity_type='Server'> \n";
print "<attributes> \n";
print "<attribute> \n";
print "<name>preferred</name> \n";
print "<value>true</value> \n";
print "</attribute> \n";
print "</attributes> \n";
print "</entity> \n";
print "<entity entity_name='stbdq17' entity_type='Server'> \n";
print "<attributes> \n";
print "<attribute> \n";
print "<name>preferred</name> \n";
print "<value>false</value> \n";
print "</attribute> \n";
print "</attributes> \n";
print "</entity> \n";
print "</entities> \n";
print "</body> \n";
print "</result> \n";

return 1;

}

# name : hasadm_get_file_content
# desc : fn to return the content of a file given file name
#
# arg  :
#  file name with whole path
# return:
#   1 for sucess
#   else failure
#
sub hasadm_get_file_content($)
{
  my ( $file_url ) = @_;

  # log the message to the log file
  warn("DEBUG:hasadm_o.pl:hasadm_get_file_content");

  warn "ERROR:hasadm_o.pl:hasadm_get_file_content File name is null\n"
   and return unless $file_url;
  # and hasadm_exitfail() and return unless $file_url;

  warn "ERROR:hasadm_o.pl:hasadm_get_file_content Failed to access file $file_url\n"
    and return unless has::Common::hasIsReadable($file_url);
    #and hasadm_exitfail() and return unless has::Common::hasIsReadable($file_url);

  my $file_content;
  $file_content = has::Common::hasReturnFileContents($file_url);

  warn "WARN:hasadm_o.pl:hasadm_get_file_content: File $file_url has no content\n" unless $file_content;

  # put filecontent in cdata section to take care of malformed xml
  # <![CDATA[$file_content]]>

  $file_content = '' unless $file_content;
  $file_content = '<![CDATA['.$file_content.']]>' if $file_content;

  my $xml_file_content_ref  = has::Common::make_element('filecontent');
  my $xml_fname_ref         = has::Common::make_element('name',$file_url);
  my $xml_fcontent_ref      = has::Common::make_element('content',$file_content);

  has::Common::append_element($xml_file_content_ref,$xml_fname_ref);
  has::Common::append_element($xml_file_content_ref,$xml_fcontent_ref);

  hasadm_xmlretcode('Success');

  my $retxmlref = hasadm_build_em_xml($xml_file_content_ref);

  has::Common::print_xml($hasadm_xml_doctype,$retxmlref);

  return 1;

}


# name : hasadm_internalresource_types
# desc : fn to return the list of internal resource types
#
# arg  :
#
# return:
#   1 for sucess
#   else failure
#
sub hasadm_internalresource_types
{
  # log the message to the log file
  warn("DEBUG:hasadm_o.pl:hasadm_internalresource_types");


  for my $inres (  qw ( oracle_database oracle_listener cluster ) )
  {
    print "em_result=$inres|$inres\n";
  }

  return 1;

}




# name : hasadm_create_action_script
# desc : fn to create an action script and copy it to all nodes of cluster
#
# arg  : file name, file contents, nodes to be copied on to
#
# return:
#   1 for sucess
#   else failure
#
sub hasadm_create_action_script
{
  # log the message to the log file
  #warn("DEBUG:hasadm_o.pl:hasadm_create_action_script");

  my $fileName = shift;
  my $fileContents = shift;
  my $n= shift;
  my @nodes= split(/,/,$n) if $n;
  my $overwrite = shift;
  my $localNode ;

  #warn "ERROR:hasadm_o.pl:hasadm_create_action_script fileName is not passed " and return 0 unless $fileName;
  print "ERROR fileName is not passed \n" and return 0 unless $fileName;
  #warn "ERROR:hasadm_o.pl:hasadm_create_action_script nodelist is not passed " and return 0 unless $n;
  #warn "ERROR:hasadm_o.pl:hasadm_create_action_script No Nodelist for creating action script " and return 0 unless @nodes;

  $overwrite = 0 unless $overwrite;

  my %command_args = (exit_failure_list => [()]);
  my $cmd = "$ENV{CRS_HOME}/bin/olsnodes -l";
  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  $localNode= has::Common::runsystemcommand($cmd,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  if ( $command_args{command_return_status} )
  {
    #warn "ERROR:hasadm_o.pl:hasadm_create_action_script Failed to get local nodename from command $cmd ";
    print "Failed to get local nodename from command $cmd \n";
    undef $localNode if defined $localNode;
  }

  chomp($localNode) if $localNode;
  $localNode = '' unless $localNode;

  my($volume,$directories,$file) = File::Spec->splitpath($fileName);
  $volume = '' unless $volume;
  $directories = '' unless $directories;
  $file = '' unless $file;

  #warn "DEBUG:hasadm_o.pl:hasadm_create_action_script volume is $volume and directory is $directories and file is $file \n";

  my $fileCheck = 1;

  stat $fileName;

  if( -f $fileName and !$overwrite) {
    #warn "ERROR:hasadm_o.pl:hasadm_create_action_script $fileName already exists on node $localNode \n";
    print "ERROR $fileName already exists on node $localNode \n";
    return 0;
  }

  open FILEHANDLE, ">$fileName" or $fileCheck=0;

  if(!$fileCheck){

    # can be a dir creation issue or dir perm issue or a fileperm issue
    my @err_list;

    if ( $directories )
    {
      stat $directories;
  
      if(!(-e $directories)){
          my $err_list;
          mkpath($directories,{ error  => \$err_list }); 
           
           if (@$err_list) {
              for my $diag (@$err_list) {
                  my ($file, $message) = each %$diag;
                  print "ERROR creating $file on node $localNode: $message\n";
              }
              #warn "ERROR:hasadm_o.pl:hasadm_create_action_script Failed to create $directories: $@ \n";
              print "ERROR in creating action script : Failed to create $directories \n";
              return 0;
           }
      }

      if(!(-w $directories)){
              #warn "ERROR:hasadm_o.pl:hasadm_create_action_script $directories do not have write permission on node $localNode \n";
              print "ERROR $directories do not have write permission on node $localNode \n";
              return 0;
       }
  

      if(!(-e $directories)){
          #warn "ERROR:hasadm_o.pl:hasadm_create_action_script  Failed creating $directories on node $localNode \n";
          print "ERROR Failed creating $directories on node $localNode \n";
          return 0 ;
      }

      stat $directories;
    }

    stat $fileName;

    if( (-f $fileName) && !(-w $fileName)){
    	#warn "ERROR:hasadm_o.pl:hasadm_create_action_script  $fileName on node $localNode doesnt have write permissions \n";
    	print "ERROR $fileName on node $localNode doesnt have write permissions \n";
        return 0;
    }

    open FILEHANDLE, ">$fileName";

  }

  print FILEHANDLE "$fileContents";

  #warn "DEBUG:hasadm_o.pl:hasadm_create_action_script FILEHANDLE fileContents";

  system("chmod 777 $fileName");

  stat $fileName;

  #warn( "DEBUG:hasadm_o.pl:hasadm_create_action_script  $fileName created") if -f $fileName;

  return 1;
}


# name : hasadm_save_action_script_to_all_nodes
# desc : fn to  copy action Script to given  nodes of cluster
#
# arg  : file name, array of nodeNames to copy to
#
# return:
#   1 for sucess
#   else failure
#
sub hasadm_save_action_script_to_all_nodes
{

  my ($fileName,$t,$overwrite) = @_;
  my @nodes = split(/,/,$t) if $t;
  $overwrite = 0 unless $overwrite;

  #warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes fileName is undefined for copying to other nodes" and return 0 unless $fileName;
  #warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes nodelist is not passed " and return 0 unless $t;
  #warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes No Nodelist " and return 0 unless @nodes;
  print "ERROR fileName is undefined for copying to other nodes\n" and return 0 unless $fileName;
  print "ERROR nodelist is not passed \n" and return 0 unless $t;
  print "ERROR No Nodelist \n" and return 0 unless @nodes;

  my %command_args = (exit_failure_list => [()]);
  my $cmd = "$ENV{CRS_HOME}/bin/olsnodes -l";
  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $localNode= has::Common::runsystemcommand($cmd,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  if ( $command_args{command_return_status} )
  {
    #warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes In retrieving master agent node name from command $cmd. Failed to save action script to all the nodes of the cluster\n";
    print "ERROR In retrieving master agent node name from command $cmd. Failed to save action script to all the nodes of the cluster\n";
    return 0;
  }

  chomp($localNode) if $localNode;

  #warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes In retrieving master agent node name from command $cmd. Failed to save action script to all the nodes of the cluster\n" unless $localNode;
  print "ERROR in retrieving master agent node name. Failed to save action script to all the nodes of the cluster\n" unless $localNode;
  warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes error in retrieving localNode information from command $cmd" unless $localNode;
  return 0 unless $localNode;

  stat $fileName;

  if( !(-f $fileName)){
  	#warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes $fileName doesnt exist on $localNode for copying to other nodes";
  	print "ERROR $fileName doesnt exist on $localNode for copying to other nodes\n";
  	return 0;
  }

  if( !(-r $fileName)){
  	#warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes $fileName doesnt have read permissions on $localNode for copying to other nodes";
  	print "ERROR $fileName doesnt have read permissions on $localNode for copying to other nodes\n";
  	return 0;
  }

  my ($volume,$directories,$file) = File::Spec->splitpath($fileName);

  $volume = '' unless $volume;
  $directories = '' unless $directories;
  $file = '' unless $file;

  # remove the hostNode for the nodes arry to be copied onto.
  if ( $localNode )
  {
    my @nlist;
    for my $np ( @nodes )
    {
      #warn "DEBUG:hasadm_o.pl:hasadm_save_action_script_to_all_nodes remove localNode $localNode from passed Node List" and next if $np =~ /^$localNode$/;
      next if $np =~ /^$localNode$/;
      push @nlist,$np;
    }
    @nodes = @nlist;
  }

  my $strg = Dumper \@nodes;
  $strg =~ s/\n//g;
  $strg = '' unless $strg;
 
  #warn "DEBUG:hasadm_o.pl:hasadm_save_action_script_to_all_nodes nodes now are : $strg and overwrite is $overwrite \n";

  my $isShared = isSharedPath($directories, \@nodes,1);
  if( $isShared){
    return 1;
  }


  my $cum_err = '';
  my %failed_node_list;
  my %file_exists_list;
  foreach my $node (@nodes) {
    my $err;
    my $remoteFileName = getRemoteFileName( $fileName, $node );

    ( my $tnsFileExist, $err ) = fileExistOnHost( $node, $fileName );

    $tnsFileExist=0 unless $tnsFileExist;
    if ( not $err or $err eq '0') {
     $err = '';
    }

    #warn "DEBUG::hasadm_o.pl:hasadm_save_action_script_to_all_nodes after checking fileExistence on host $node for $fileName file existence - $tnsFileExist and error = $err \n";

    if ($tnsFileExist && !$overwrite) {
     $cum_err = $cum_err." $fileName already exists on host $node" ;
     $file_exists_list{$node}=1;
    } else {
	if(!$tnsFileExist){
          #warn "DEBUG:hasadm_o.pl:hasadm_save_action_script_to_all_nodes file $fileName doesnt exist on $node \n";
          ( my $ret, $err) = prepareForCopy($fileName,$node);
          if(defined($err) && $err ne ''){
            $cum_err .= " Error on node: $node".$err;
            $failed_node_list{$node}=1;
          }
        }
    }
  }

  chomp($cum_err) if $cum_err;
  if( $cum_err ){
    #warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes in copying actionscript to other nodes err=$cum_err";
    print "ERROR in copying actionscript to other nodes err=$cum_err\n";
    return ('0');
  }

  foreach my $node (@nodes) {
    my $remoteFileName = getRemoteFileName( $fileName, $node );
    #warn("DEBUG:hasadm_o.pl:hasadm_save_action_script_to_all_nodes executing remoteCopy on $node for file $fileName \n");
    my $opCode = executeRemoteCopy( $fileName, $remoteFileName );
    if(not $opCode or $opCode eq '0'){
      #warn "ERROR:hasadm_o.pl:hasadm_save_action_script_to_all_nodes in copying actionscript to the node $node";
      print "ERROR in copying actionscript to the node $node\n";
    }
  }

  return (1);

}


# name : prepareForCopy
# desc : branched out code for hasadm_save_action_script_to_all_nodes
#        a. checks if the directory exists b. if not creates the directory or returns error
# args  :
#   i) fileName
# return:
#
sub prepareForCopy
{
	my $fileName = shift;
	my $host = shift;
	my $err = '';
	my ( $ssh_cmd, $rsh_cmd )    = ( "/usr/bin/ssh", "/usr/bin/rsh" );
	my ( $visible, $notvisible ) = ( "dirVisible",  "notDirVisible" );
	my ( $writable, $notwritable) = ( "dirWritable",  "notDirWritable" );
	my %command_args = (exit_failure_list => [()]);

        $fileName = '' unless $fileName;
        $host = '' unless $host;
	#warn"DEBUG:hasadm_o.pl:prepareForCopy: fileName is $fileName";
	my($volume,$dir,$file) = File::Spec->splitpath($fileName);

        $volume = '' unless $volume;
        $dir = '' unless $dir;
        $file = '' unless $file;

	my $commandForDirCheck = " $host -n /bin/sh  -c \'\" if [ -d $dir ] ; then echo $visible; else echo $notvisible;  fi \"\' ";
	my $commandForDirPermCheck = " $host -n /bin/sh  -c \'\" if [ -w $dir ] ; then echo $writable; else echo $notwritable;  fi \"\' ";
	my $commandForDirCreate = " $host -n  \'mkdir -p $dir\' ";

        my $cmd = '';
    	if ( -e $ssh_cmd ) {
	        # assume success op
	        $cmd = "$ssh_cmd ";
	        $cmd .= $commandForDirCheck;
	        ( $err, my $output ) = executeCommand($cmd);
                $err = '' unless $err;
                if ( $output )
                {
                  chomp($output);
                }
                else
                {
	          #warn"DEBUG:hasadm_o.pl:prepareForCopy rac_services_TNS_o:fileExistOnHost = Failed executing command $cmd err=$err ";
                  $output = '';
                }
                #warn"DEBUG:hasadm_o.pl:prepareForCopy executed commandForDirCheck error is $err and output is $output\n";
	        #$retVal = -1   if $err ne "" or ( $output ne "" && !( $output =~ /\s*($visible|$notvisible)\s*/ ) );
	        #warn"DEBUG:hasadm_o.pl:prepareForCopy rac_services_TNS_o:fileExistOnHost output = $output err=$err";
		if((defined($err) && $err ne '') || ($output eq $notvisible)){
  		  #create the directory
  	          $cmd = "$ssh_cmd ";
  	          $cmd .= $commandForDirCreate;
                  #warn"DEBUG:hasadm_o.pl:prepareForCopy before executing commandForDirCreate node is $host cmd is $cmd \n";
  		  # ( my $errCreateDir, $output ) = executeCommand($cmd);
                  %command_args = (exit_failure_list => [()]);
  		  my $errCreateDir = has::Common::runsystemcommand($cmd,"",\%command_args);
  
                  if ( $command_args{command_return_status} )
                  {
                    my $errmsg = $command_args{command_error_message_stderr} 
                     if $command_args{command_error_message_stderr};
                    $errmsg = $errCreateDir if $errCreateDir and not $errmsg;

                    $errmsg = '' unless $errmsg;

  	            print "ERROR  Failed to create directory $dir $errmsg on node $host\n";
  	            #warn "ERROR:hasadm_o.pl:prepareForCopy Failed to create directory $dir $errmsg\n";
  		    return (0,$errmsg);
                  }

                } else {
                # the directory exists. now check if it is writable
                $cmd = "$ssh_cmd ";
                $cmd .= $commandForDirPermCheck;
                ( $err, my $output ) = executeCommand($cmd);
                    $err = '' unless $err;
                    if ( $output )
                    {
                      chomp($output);
                    }
                    else
                    {
                  #warn"DEBUG:hasadm_o.pl:prepareForCopy rac_services_TNS_o:fileExistOnHost = Failed executing command $cmd err=$err ";
                      $output = '';
                    }
    			if((defined($err) && $err ne '') || ($output eq $notwritable)){
    			    return (0," $dir does not have write permissions."); 
    			}

            }
		}elsif ( -e $rsh_cmd ) {
	        # assume success op
	        $cmd = "$rsh_cmd ";
	        $cmd .= $commandForDirCheck;
	        ( $err, my$output ) = executeCommand($cmd);
                $err = '' unless $err;
                if ( $output )
                {
                  chomp($output);
                }
                {
	          #warn "DEBUG:hasadm_o.pl:prepareForCopy rac_services_TNS_o:fileExistOnHost = Failed executing command $cmd err=$err";
                  $output = '';
                }
                #warn"DEBUG:hasadm_o.pl:prepareForCopy executed commandForDirCheck error is $err and output is $output\n";
	        #$retVal = -1   if $err ne "" or ( $output ne "" && !( $output =~ /\s*($visible|$notvisible)\s*/ ) );
	        #warn "DEBUG:hasadm_o.pl:prepareForCopyrac_services_TNS_o:fileExistOnHost output = $output err=$err " ;
			if((defined($err) && $err ne '') || ($output eq $notvisible)){
				#create the directory
	            $cmd = "$rsh_cmd ";
	            $cmd .= $commandForDirCreate;
                    #warn"DEBUG:hasadm_o.pl:prepareForCopy before executing commandForDirCreate node is $host cmd is $cmd \n" ;
                  %command_args = (exit_failure_list => [()]);
                  my $errCreateDir = has::Common::runsystemcommand($cmd,"",\%command_args);

                  if ( $command_args{command_return_status} )
                  {
                    my $errmsg = $command_args{command_error_message_stderr}
                     if $command_args{command_error_message_stderr};
                    $errmsg = $errCreateDir if $errCreateDir and not $errmsg;

                    $errmsg = '' unless $errmsg;

                    print "ERROR  Failed to create directory $dir $errmsg\n";
                    #warn "ERROR:hasadm_o.pl:prepareForCopy Failed to create directory $dir $errmsg\n";
                    return (0,$errmsg);
                  }

            }else {
                # the directory exists. now check if it is writable
	            $cmd = "$rsh_cmd ";
                $cmd .= $commandForDirPermCheck;
                ( $err, my $output ) = executeCommand($cmd);
                    $err = '' unless $err;
                    if ( $output )
                    {
                      chomp($output);
                    }
                    else
                    {
                  #warn"DEBUG:hasadm_o.pl:prepareForCopy rac_services_TNS_o:fileExistOnHost = Failed executing command $cmd err=$err ";
                      $output = '';
                    }
    			if((defined($err) && $err ne '') || ($output eq $notwritable)){
    			    return (0," $dir does not have write permissions.");
    			}

            }
		}
	return (1,'');
}

# name : hasadm_create_and_start
# desc : fn to create and start a resource
#
# args  :
#   i) command to create resource
#  ii) resource name
# return:
#
sub hasadm_create_and_start_new
{
  my ($resCreateCommand, $resName, $username, $password, $usesu) = @_;
  my $resStartCommand = "crsctl start resource " . $resName;

  my %run_cmd_results;

  if($usesu)
  {
    hasadm_run_crs_command_su($username, $password,$resCreateCommand,\%run_cmd_results);
  }
  else
  {
    hasadm_run_crs_command($resCreateCommand,\%run_cmd_results);
  }

  print "Error executing command $resCreateCommand\n" if $run_cmd_results{command_return_status};

  if( keys %run_cmd_results and $run_cmd_results{cmd_results} and $run_cmd_results{cmd_results} =~ /CRS-4000/) {
    # indicates create failed. hence return
    print "em_multi_op_result=1"; # to indicate creation failed
    #replace new line characters with space if any
    print $run_cmd_results{cmd_results} if $run_cmd_results{cmd_results};
    return 1;
  }

  my $cmdresults1 = $run_cmd_results{cmd_results} if $run_cmd_results{cmd_results};
  $cmdresults1 = '' unless $cmdresults1;

  %run_cmd_results=();
  hasadm_run_crs_command($resStartCommand,\%run_cmd_results);

  print "Error executing command $resStartCommand\n" if $run_cmd_results{command_return_status};

  $cmdresults1 = "$cmdresults1 $run_cmd_results{cmd_results}" if $run_cmd_results{cmd_results};

  print "$cmdresults1" if $cmdresults1;

  return 1;

}


sub hasadm_create_and_start
{
  my ($resCreateCommand, $resName, $username, $password, $usesu) = @_;
  my $resStartCommand = "crsctl start resource " . $resName;

  my %command_args = (exit_failure_list => [()]);

  if($usesu)
  {
    if ($username =~ /^aime/)
    {
      $resCreateCommand = "/usr/local/packages/aime/em/run_as_root '" . $resCreateCommand . "'";
      #$resStartCommand = "/usr/local/packages/aime/em/run_as_root '" . $resStartCommand . "'";
    }
    else
    {
      # actual command
      $resCreateCommand = "echo $password | sudo -S " . $resCreateCommand;
      # execute sudo -k to clear sudo priviliges
      $resCreateCommand = $resCreateCommand . ";sudo -k";

      # actual command
      #$resStartCommand = "echo $password | sudo -S " . $resStartCommand;
      # execute sudo -k to clear sudo priviliges
      #$resStartCommand = $resStartCommand . ";sudo -k";

    }
  }

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $cmdresults = has::Common::runsystemcommand($resCreateCommand,"",\%command_args);
  has::Common::hasRestoreCRSEnv();

  print "Error executing command $resCreateCommand\n" if $command_args{command_return_status};

  #strip beginning and trailing spaces
  $cmdresults =~ s/^\s+|\s+$//g if $cmdresults;

  #replace new line characters with space if any
  $cmdresults =~ s/\n+/ /g if $cmdresults;

  if($cmdresults =~ /CRS-4000/) {
    # indicates create failed. hence return
    print "em_multi_op_result=1"; # to indicate creation failed
    print $cmdresults;
    return 1;
  }

  %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($ENV{CRS_HOME});
  my $scmdresults = has::Common::runsystemcommand($resStartCommand, "",\%command_args);
  has::Common::hasRestoreCRSEnv();

  print "Error executing command $resStartCommand\n" if $command_args{command_return_status};

  #strip beginning and trailing spaces
  $scmdresults =~ s/^\s+|\s+$//g if $scmdresults;

  #replace new line characters with space if any
  $scmdresults =~ s/\n+/ /g if $scmdresults;

  $cmdresults = $cmdresults . " " . $scmdresults;

  print $cmdresults;

  return 1;

}



# name : hasadm_get_dependents_multiple_resources
# desc : fn to get dependencies information for multiple resources
#
# arg  :
#  list of resource names separated by comma
# return:
#
sub hasadm_get_dependents_multiple_resources
{
  my $resList = shift; #list of comma separated resource names
  my @resListArray = split/,/,$resList;

  for my $res (@resListArray) {
    print $res;
    print "ORACLE_EM_RESOURCE_NAME_SEPARATOR";
    my $command = "emcrsp em get_dependents -e resource -a $res";
    hasadm_main('hasadm_run_crs_command', $command);
    print "ORACLE_EM_NEXT_RESOURCE_SEPARATOR";
  }
  return 1;
}

# name : hasadm_register_listener
# desc : function to register listener
#
# arg  :
#   listener name
#   port
#   oracle home for listener
#
# return:
#
sub hasadm_register_listener
{
  my ( $lsnrName,$port,$lsnrHome ) = @_;

  my %command_args = (exit_failure_list => [()]);

  has::Common::hasSetCRSEnv($lsnrHome);
  my $cmdresults = has::Common::runsystemcommand('lsnrctl'," stat $lsnrName",\%command_args);
  has::Common::hasRestoreCRSEnv();

  my $arg = " add listener -l $lsnrName -p $port -o $lsnrHome ";
  if ( not $command_args{command_return_status} )
  {
    $arg = "$arg -s ";
  }
  

  %command_args = (exit_failure_list => [()]);
  has::Common::hasSetCRSEnv($lsnrHome);
  $cmdresults = has::Common::runsystemcommand('srvctl',$arg,\%command_args);
  has::Common::hasRestoreCRSEnv();

  if ( $command_args{command_return_status}  )
  {
    warn "ERROR:hasadm_o.pl:hasadm_register_listener:Failed to register the listener\n";
  }

  #strip beginning and trailing spaces
  $cmdresults =~ s/^\s+|\s+$//g if $cmdresults;

  #replace new line characters with space if any
  $cmdresults =~ s/\n+/ /g if $cmdresults;

  print $cmdresults;

  return 1;
}

#------------------------------------------------------------------------------
# FUNCTION :    hasadm_res_dep_strg
#
# DESC
#  get the dependencies and the dependents as a nested string for each resource
#
# ARGUMENTS
#  NULL
#------------------------------------------------------------------------------
sub hasadm_res_dep_strg
{

  # get the coma seperated list of crs nodes
  my $res_ref = has::Common::hasGetResourceDependencyString($ENV{CRS_HOME});

  if ( $res_ref and ref($res_ref) and ref($res_ref) =~ /HASH/i )
  {
    for my $key ( keys %{$res_ref} )
    {
        print "em_result=$key|";
        if ( $res_ref->{$key}{dependent_on} )
        {
           print "$res_ref->{$key}{dependent_on}";
        }
        print "|";
        if ( $res_ref->{$key}{dependents} )
        {
           print "$res_ref->{$key}{dependents}";
        }
        print "\n";
    }
  }

  return 1;

}


#------------------------------------------------------------------------------
# FUNCTION :    hasadm_get_dependents_couple
#
# DESC
#  get the dependencies as parent child couples
#
# ARGUMENTS
#  NULL
#------------------------------------------------------------------------------
sub hasadm_get_dependents_couple
{

  # get the coma seperated list of crs nodes
  my $res_ref = has::Common::hasGetResourceDependents($ENV{CRS_HOME});

  if ( $res_ref and ref($res_ref) and ref($res_ref) =~ /HASH/i )
  {
    for my $key ( keys %{$res_ref} ) 
    {
      for my $dep ( keys %{$res_ref->{$key}} )
      {
        print "em_result=$key|$dep";
        if ( $res_ref->{$key}{$dep} )
        {
           print "|$res_ref->{$key}{$dep}";
        }
        else
        {
           print "|";
        }

        print "\n";
      }
    }
  }

  return 1;

}
#------------------------------------------------------------------------------
# FUNCTION :    hasadm_main
#
# DESC
# start the function
#
# ARGUMENTS
# function name
# args to function
#------------------------------------------------------------------------------
my %hasadm_fnlist;
my %hasadm_format;

$hasadm_fnlist{hasadm_start_nodes}=\&hasadm_start_nodes;
$hasadm_format{hasadm_start_nodes}='std';

$hasadm_fnlist{hasadm_stop_nodes}=\&hasadm_stop_nodes;
$hasadm_format{hasadm_stop_nodes}='std';

$hasadm_fnlist{hasadm_check_nodes}=\&hasadm_check_nodes;
$hasadm_format{hasadm_check_nodes}='std';

$hasadm_fnlist{hasadm_get_nodename}=\&hasadm_utl_get_nodename;

$hasadm_fnlist{hasadm_nodes}=\&hasadm_nodes;
$hasadm_format{hasadm_nodes}='std';

$hasadm_fnlist{hasadm_crsnodes}=\&hasadm_crsnodes;
$hasadm_format{hasadm_crsnodes}='xml';

$hasadm_fnlist{hasadm_resources}=\&hasadm_resources;
$hasadm_format{hasadm_resources}='xml';

$hasadm_fnlist{hasadm_resources_combined}=\&hasadm_resources_combined;
$hasadm_format{hasadm_resources_combined}='xml';

$hasadm_fnlist{hasadm_server_groups}=\&hasadm_server_groups;
$hasadm_format{hasadm_server_groups}='xml';

$hasadm_fnlist{hasadm_server_groups_combined}=\&hasadm_server_groups_combined;
$hasadm_format{hasadm_server_groups_combined}='xml';

$hasadm_fnlist{hasadm_run_crs_command}=\&hasadm_run_crs_command;
$hasadm_format{hasadm_run_crs_command}='no_error_print';

$hasadm_fnlist{hasadm_create_and_start}=\&hasadm_create_and_start;
$hasadm_format{hasadm_create_and_start}='no_error_print';

$hasadm_fnlist{hasadm_run_crs_command_su}=\&hasadm_run_crs_command_su;
$hasadm_format{hasadm_run_crs_command_su}='no_error_print';

$hasadm_fnlist{hasadm_servers}=\&hasadm_servers;
$hasadm_format{hasadm_servers}='xml';

$hasadm_fnlist{hasadm_resource_types}=\&hasadm_resource_types;
$hasadm_format{hasadm_resource_types}='xml';

$hasadm_fnlist{hasadm_resourceinstances}=\&hasadm_resourceinstances;
$hasadm_format{hasadm_resourceinstances}='xml';

$hasadm_fnlist{hasadm_allnodes_with_crsstatus}=\&hasadm_allnodes_with_crsstatus;
$hasadm_format{hasadm_allnodes_with_crsstatus}='std';

$hasadm_fnlist{hasadm_allnode_crsstatus}=\&hasadm_allnode_crsstatus;
$hasadm_format{hasadm_allnode_crsstatus}='std';

$hasadm_fnlist{hasadm_get_file_content}=\&hasadm_get_file_content;
$hasadm_format{hasadm_get_file_content}='xml';

$hasadm_fnlist{hasadm_internalresource_types}=\&hasadm_internalresource_types;
$hasadm_format{hasadm_internalresource_types}='std';

$hasadm_fnlist{hasadm_create_action_script}=\&hasadm_create_action_script;
$hasadm_format{hasadm_create_action_script}='no_error_print';

$hasadm_fnlist{hasadm_save_action_script_to_all_nodes}=\&hasadm_save_action_script_to_all_nodes;
$hasadm_format{hasadm_save_action_script_to_all_nodes}='no_error_print';

$hasadm_fnlist{hasadm_get_resource_dependency_string}=\&hasadm_res_dep_strg;
$hasadm_format{hasadm_get_resource_dependency_string}='std';

$hasadm_fnlist{hasadm_get_dependents_multiple_resources}=\&hasadm_get_dependents_multiple_resources;
$hasadm_format{hasadm_get_dependents_multiple_resources}='xml';

$hasadm_fnlist{hasadm_get_resource_dependents}=\&hasadm_get_dependents_couple;
$hasadm_format{hasadm_get_resource_dependents}='std';

$hasadm_fnlist{hasadm_register_listener}=\&hasadm_register_listener;
$hasadm_format{hasadm_register_listener}='std';

sub hasadm_main($;@)
{
  my ($fnname,@args) = @_;

  chomp $fnname;

  $fnname =~ s/^\s+|\s+$//g;

  die "ERROR:hasadm_o.pl:hasadm_main function name is required to be passed to hasadm_main\n"
   unless $fnname;

  die "ERROR:hasadm_o.pl:hasadm_main Function name $fnname is not declared in hasadm_o.pl\n"
   unless $hasadm_fnlist{$fnname} and defined &{$hasadm_fnlist{$fnname}};

  #initalization
  hasadm_init();

  # the format of the results and the error printing function to be used
  $hasadm_output_format = $hasadm_format{$fnname} if $hasadm_format{$fnname};

#-----------------------------------------------------------
#  this block is for debug in dev env it will be delted
#-----------------------------------------------------------
if ( $DEV_DEBUG )
{
  my $filedebug = '/tmp/hasadmdbg.txt';
  my $tm = localtime;
  open(FHD,'>>',$filedebug) or die "Failed to open debug file $filedebug\n";
  print FHD "Invoked at $tm for function $fnname ";
  for my $x ( @args )
  {
     print FHD "arg=$x , ";
  }

  if ( $ENV{CRS_HOME} )
  {
     print FHD "\nInvoked at $tm for function $fnname with CRS_HOME $ENV{CRS_HOME}\n";
  }
  elsif ( $ENV{ORACLE_HOME} )
  {
     print FHD "\nInvoked at $tm for function $fnname with ORACLE_HOME $ENV{ORACLE_HOME}\n";
  }

  print FHD "\n";
} 

  # invoke the function
  my $retval;
  $retval = $hasadm_fnlist{$fnname}(@args);

  if ( not $retval )
  {
     warn "ERROR:hasadm_o.pl:hasadm_main Failed to execute the function $fnname\n" unless $hasadm_output_format =~ /no_error_print/;
  }

  # print error/warnings as em_warnings to stdout
  #  only for std output, xml out put should be doing this in the functions
  if ( $hasadm_output_format =~ /std/ )
  {
    has::Common::has_printerrors()  or die "ERROR:Failed to print processing errors/warnings for $fnname\n";
  }

  hasadm_cleanup();

if ( $DEV_DEBUG )
{
  close(FHD);
}

  return $retval;

}

#hasadm_main('hasadm_resources','');
#hasadm_main('hasadm_server_groups_combined');
#my @n = ('stbdq12','stbdq13','stbdq14');
#hasadm_main('hasadm_save_action_script_to_all_nodes','/scratch/aime/public/newDir1/delMea.pl','stbdq12,stbdq13,stbdq14',1);
#hasadm_main('hasadm_create_action_script','/scratch/vthondep/public/delMe.pl','Delete Me please','stdbq12,stbdq13,stbdq14');
#----------------------------------------------------------------------------------------------
#  The code below this for testing purposes - it will be deleted
#----------------------------------------------------------------------------------------------

#sub hasadm_crs_hosts($)
#{
# return 1;
#}


#sub hasadm_crs_resource_props($)
#{
#  return 1;
#}