Expect - can not find channel named "" while executing

Trying to spawn an SSH session to 1 or more of a branch locations devices, from within a single expect script.

I have a bash script which takes 3 args:

1. the Expect script
2. the Output directory for the results of the expect script
3. the list of branch locations that i want to connect to

All the devices are mapped to DNS nodes, so site 1234 will have 1-5 devices I want. Lets say like 1234R, 1234FW1, 1234S1, and/or 1234S2.

The list of sites is pulled from a file passed to the bash wrapper. The site number is passed to the expect script from this file.

The expect script has 5 list objects, and 5 if statementes. If the list is not empty, process the commands for in the list. Each list, corresponds to a device type I want to connect to at a branch location (They have to be different because the CLI prompt is different, and we don't want to accidentally run the commands on a device that we did not intend to)

My problem is that, if my expect script is unable to process device 2/5, say because it is unreachable/offline, it stops processing the rest.

So if for site 1234 I wanted to process commands on device 2/5 (FW1) and device 4/5 (S1) but FW1 is offline, S1 is never processed.

I can't exit because that exits the entire script. I've tried using expect eof, close -i $spawn_id to close the global spawn_id, and even a timeout, but the process just gives me the same error everytime. I feel like I'm missing something super obvious.

Here's the Expect script:

#!/usr/bin/expect

#Purpose: SSH to remote juniper hosts and execute commands.
#Syntax: <script name> <site number> <username> <password>

global spawn_id;
global sess_id;


set cmds_R [ list \

]


set cmds_FW1 [ list \
"show system license" \
"exit" \
]


set cmds_10MB [ list \

]


set cmds_S1 [ list \
"show interfaces terse | no-more" \
"exit" \
]


set cmds_S2 [ list \

]













# Check for Primary Router commands
# If found, run Primary router commands.

if { $cmds_R != "" } {
  proc _jgetRouterIPs {} {
  set file [open router-list r]

while {![eof $file]} {
  set buff [read $file]
}
close $file
return $buff
}



proc ssh2rtr { rtr usr pwd } {
    spawn ssh -l $usr $rtr;
    set sess_id $spawn_id;
    set ppret [ processPrompts $rtr $usr $pwd $sess_id 0 ];
    set timeout 60;
if { $ppret == 0 } { 
    return $spawn_id; 
} else { 
    send_user "Failed to process prompts on R!!!";
  }
}


proc processPrompts { rtr usr pwd sess_id status } {
      set timeout 90;
      set spawn_id $sess_id;
      set fw_prompt  {\n*@[0-9]{4}\-[A-Z]{2}\-R(>|#)\s};

if { $status < 2 } {
      expect {
      -re  $fw_prompt { return 0;}

      "no)?" {  send "yes\r";
       processPrompts $rtr $usr $pwd $sess_id 0;
        }


      "assword:" {
                send "${pwd}\r";
       if { $status} { return 1;}
       processPrompts $rtr $usr $pwd $sess_id 1;
        }

      "ssh: Could not resolve hostname" {
          send_user "\nDNS Error!!! Check Hostname and try again!\n"; return 10;
      }


     timeout     { send_user "\nTimeout on SSH!!!\n"; return 8;}  ;#no reponse from telnet/ssh
     default     { send_user "\nSSH Unknown Error!!!\n"; return 9;} ;#unknown error has occurred.
} ;#end expect
} ;#end if
}


set site [lindex $argv 0];
set fw "${site}R";
set usr [lindex $argv 1];
set pwd [lindex $argv 2];
set fw_prompt  {\n*@[0-9]{4}\-[A-Z]{2}\-R(>|#)\s};
set yesorp {(\(yes\))|(-R(>|#))};


set timeout 90;
set sshret  [ ssh2rtr $fw $usr $pwd ] ;
if { $sshret != 99 } {
    set spawn_id $sshret;
} else {
    send_user "Error Connecting...!!!";
}

foreach cmd $cmds_R {
send "${cmd}\r";
expect -re $fw_prompt;
}

} else {
  send_user "\nNo commands for Router\n";
}


















# Check for Backup Router commands
# If found, run Backup router commands.


if { $cmds_FW1 != "" } {
proc _jgetRouterIPs {} {
  set file [open router-list r]

while {![eof $file]} {
  set buff [read $file]
}
close $file
return $buff
}



proc ssh2rtr { rtr usr pwd } {
    spawn ssh -l $usr $rtr;
    set sess_id $spawn_id;
    set ppret [ processPrompts $rtr $usr $pwd $sess_id 0 ];
    set timeout 60;
if { $ppret == 0 } { 
    return $spawn_id; 
} else { 
    send_user "Failed to process prompts on FW1!!!"; 
  }
}


proc processPrompts { rtr usr pwd sess_id status } {
      set timeout 90;
      set spawn_id $sess_id;
      set fw_prompt  {\n*@[0-9]{4}\-[A-Z]{2}\-FW1(>|#)\s};

if { $status < 2 } {
      expect {
      -re  $fw_prompt { return 0;}

      "no)?" {  send "yes\r";
       processPrompts $rtr $usr $pwd $sess_id 0;
        }


      "assword:" {
                send "${pwd}\r";
       if { $status} { return 1;}
       processPrompts $rtr $usr $pwd $sess_id 1;
        }

      "ssh: Could not resolve hostname" {
          send_user "\nDNS Error!!! Check Hostname and try again!\n"; return 10;
      }


     timeout     { send_user "\nTimeout on SSH!!!\n"; return 8;}  ;#no reponse from telnet/ssh
     default     { send_user "\nSSH Unknown Error!!!\n"; return 9;} ;#unknown error has occurred.
} ;#end expect
} ;#end if
}


set site [lindex $argv 0];
set fw "${site}FW1";
set usr [lindex $argv 1];
set pwd [lindex $argv 2];
set fw_prompt  {\n*@[0-9]{4}\-[A-Z]{2}\-FW1(>|#)\s};
set yesorp {(\(yes\))|(-FW1(>|#))};


set timeout 90;
set sshret  [ ssh2rtr $fw $usr $pwd ] ;
if { $sshret != 99 } {
    set spawn_id $sshret;
} else {
    send_user "Error Connecting...!!!";
}


foreach cmd $cmds_FW1 {
send "${cmd}\r";
expect -re $fw_prompt;
}


} else {
  send_user "\nNo commands for Backup FW1\n";
}


















# Check for 10MB Router commands
# If found, run 10MB Router commands.

if { $cmds_10MB != "" } {
proc _jgetRouterIPs {} {
  set file [open router-list r]

while {![eof $file]} {
  set buff [read $file]
}
close $file
return $buff
}



proc ssh2rtr { rtr usr pwd } {
    spawn ssh -l $usr $rtr;
    set sess_id $spawn_id;
    set ppret [ processPrompts $rtr $usr $pwd $sess_id 0 ];

if { $ppret == 0 } { 
    return $spawn_id; 
} else { 
    send_user "Failed to process prompts on 10MB!!!";
    close;
    close -i $spawn_id;
  }
}


proc processPrompts { rtr usr pwd sess_id status } {
      set timeout 90;
      set spawn_id $sess_id;
      set fw_prompt  {[a-z0-9]+@+[0-9]+([A-Za-z0-9-]{5,11})(>|#)};

if { $status < 2 } {
      expect {
      -re  $fw_prompt { return 0;}

      "no)?" {  send "yes\r";
       processPrompts $rtr $usr $pwd $sess_id 0;
        }


      "assword:" {
                send "${pwd}\r";
       if { $status} { return 1;}
       processPrompts $rtr $usr $pwd $sess_id 1;
        }

      "ssh: Could not resolve hostname" {
          send_user "\nDNS Error!!! Check Hostname and try again!\n"; return 10;
      }


     timeout     { send_user "\nTimeout on SSH!!!\n"; return 8;}  ;#no reponse from telnet/ssh
     default     { send_user "\nSSH Unknown Error!!!\n"; return 9;} ;#unknown error has occurred.
} ;#end expect
} ;#end if
}


set site [lindex $argv 0];
set fw "${site}R";
set usr [lindex $argv 1];
set pwd [lindex $argv 2];
set fw_prompt  {[a-z0-9]+@+[0-9]+([A-Za-z0-9-]{5,11})(>|#)};
set yesorp {(\(yes\))|([A-Za-z0-9-]{5,11})(>|#)};


set timeout 90;
set sshret  [ ssh2rtr $fw $usr $pwd ] ;
if { $sshret != 99 } {
    set spawn_id $sshret;
} else {
    send_user "Error Connecting...!!!";
}


foreach cmd $cmds_10MB {
send "${cmd}\r";
expect -re $fw_prompt;
}


} else {
  send_user "\nNo commands for 10MB Router\n";
}



















# Check for Switch 1 commands
# If found, run Switch 1 commands.

if { $cmds_S1 != "" } {
proc _jgetRouterIPs {} {
  set file [open router-list r]

while {![eof $file]} {
  set buff [read $file]
}
close $file
return $buff
}



proc ssh2rtr { rtr usr pwd } {
    spawn ssh -l $usr $rtr;
    set sess_id $spawn_id;
    set ppret [ processPrompts $rtr $usr $pwd $sess_id 0 ];

if { $ppret == 0 } { 
    return $spawn_id; 
} else { 
    send_user "Failed to process prompts on S1!!!";
    close;
    close -i $spawn_id;
  }
}


proc processPrompts { rtr usr pwd sess_id status } {
      set timeout 90;
      set spawn_id $sess_id;
      set fw_prompt  {\n*@[0-9]{4}\-[A-Z]{2}\-S1(>|#)\s};

if { $status < 2 } {
      expect {
      -re  $fw_prompt { return 0;}

      "no)?" {  send "yes\r";
       processPrompts $rtr $usr $pwd $sess_id 0;
        }


      "assword:" {
                send "${pwd}\r";
       if { $status} { return 1;}
       processPrompts $rtr $usr $pwd $sess_id 1;
        }

      "ssh: Could not resolve hostname" {
          send_user "\nDNS Error!!! Check Hostname and try again!\n"; return 10;
      }


     timeout     { send_user "\nTimeout on SSH!!!\n"; return 8;}  ;#no reponse from telnet/ssh
     default     { send_user "\nSSH Unknown Error!!!\n"; return 9;} ;#unknown error has occurred.
} ;#end expect
} ;#end if
}


set site [lindex $argv 0];
set fw "${site}S1";
set usr [lindex $argv 1];
set pwd [lindex $argv 2];
set fw_prompt  {\n*@[0-9]{4}\-[A-Z]{2}\-S1(>|#)\s};
set yesorp {(\(yes\))|(-S1(>|#))};


set timeout 90;
set sshret  [ ssh2rtr $fw $usr $pwd ] ;
if { $sshret != 99 } {
    set spawn_id $sshret;
} else {
    send_user "Error Connecting...!!!";
}


foreach cmd $cmds_S1 {
send "${cmd}\r";
expect -re $fw_prompt;
}


} else {
  send_user "\nNo commands for Switch 1\n";
}















# Check for Switch 2 commands
# If found, run Switch 2 commands.

if { $cmds_S2 != "" } {
proc _jgetRouterIPs {} {
  set file [open router-list r]

while {![eof $file]} {
  set buff [read $file]
}
close $file
return $buff
}



proc ssh2rtr { rtr usr pwd } {
    spawn ssh -l $usr $rtr;
    set sess_id $spawn_id;
    set ppret [ processPrompts $rtr $usr $pwd $sess_id 0 ];

if { $ppret == 0 } { 
    return $spawn_id; 
} else { 
    send_user "Failed to process prompts on S2!!!";
    close;
    close -i $spawn_id;
  }
}


proc processPrompts { rtr usr pwd sess_id status } {
      set timeout 90;
      set spawn_id $sess_id;
      set fw_prompt  {\n*@[0-9]{4}\-[A-Z]{2}\-S2(>|#)\s};

if { $status < 2 } {
      expect {
      -re  $fw_prompt { return 0;}

      "no)?" {  send "yes\r";
       processPrompts $rtr $usr $pwd $sess_id 0;
        }


      "assword:" {
                send "${pwd}\r";
       if { $status} { return 1;}
       processPrompts $rtr $usr $pwd $sess_id 1;
        }

      "ssh: Could not resolve hostname" {
          send_user "\nDNS Error!!! Check Hostname and try again!\n"; return 10;
      }


     timeout     { send_user "\nTimeout on SSH!!!\n"; return 8;}  ;#no reponse from telnet/ssh
     default     { send_user "\nSSH Unknown Error!!!\n"; return 9;} ;#unknown error has occurred.
} ;#end expect
} ;#end if
}


set site [lindex $argv 0];
set fw "${site}S2";
set usr [lindex $argv 1];
set pwd [lindex $argv 2];
set fw_prompt  {\n*@[0-9]{4}\-[A-Z]{2}\-S2(>|#)\s};
set yesorp {(\(yes\))|(-S2(>|#))};


set timeout 90;
set sshret  [ ssh2rtr $fw $usr $pwd ] ;
if { $sshret != 99 } {
    set spawn_id $sshret;
} else {
    send_user "Error Connecting...!!!";
}


foreach cmd $cmds_S2 {
send "${cmd}\r";
expect -re $fw_prompt;
}


} else {
  send_user "\nNo commands for Switch 2\n";
}

The output I get to the console when a device is unreachable is:

can not find channel named ""
    while executing
"send "${cmd}\r""
    ("foreach" body line 2)
    invoked from within
"foreach cmd $cmds_FW1 {
send "${cmd}\r";
expect -re $fw_prompt;
}"
    invoked from within
"if { $cmds_FW1 != "" } {
proc _jgetRouterIPs {} {
  set file [open router-list r]

while {![eof $file]} {
  set buff [read $file]
}
close $file
return..."
    (file "./show-sys-lic_FW1" line 157)

I want to process this more cleanly and output a message via send_user saying unable to process prompts, and then move on until all 5 if statements have been tested and completed.

Currently it will do the send_user in the else clause, but anything else I add there doesn't let me proceed past that first if/else statment. The script just exits.

  if { $ppret == 0 } { 
        return $spawn_id; 
    } else { 
        send_user "Failed to process prompts on R!!!";
      }
    }