This commit is contained in:
Logan Lipke
2020-05-27 14:18:16 -07:00
5 changed files with 275 additions and 168 deletions

View File

@@ -11,36 +11,33 @@ import json
import pprint
from LANforge import LFRequest
from LANforge import LFUtils
from LANforge.LFUtils import *
import create_genlink as genl
debugOn = True
if sys.version_info[0] != 3:
print("This script requires Python 3")
exit(1)
mgrURL = "http://localhost:8080/"
def execWrap(cmd):
if os.system(cmd) != 0:
print("\nError with " + cmd + ",bye\n")
exit(1)
def jsonReq(mgrURL, reqURL, data, debug=False):
def jsonReq(mgrURL, reqURL, data, exitWhenCalled=False):
lf_r = LFRequest.LFRequest(mgrURL + reqURL)
lf_r.addPostData(data)
if debug:
json_response = lf_r.jsonPost(debug)
LFUtils.debug_printer.pprint(json_response)
if exitWhenCalled:
json_response = lf_r.jsonPost(True)
print("jsonReq: debugdie Response: ")
LFUtils.debug_printer.pprint(vars(json_response))
print("jsonReq: bye")
sys.exit(1)
else:
lf_r.jsonPost(debug)
lf_r.jsonPost(exitWhenCalled)
def getJsonInfo(mgrURL, reqURL):
def getJsonInfo(mgrURL, reqURL, debug=False):
lf_r = LFRequest.LFRequest(mgrURL + reqURL)
json_response = lf_r.getAsJson(debugOn)
json_response = lf_r.getAsJson(debug)
return json_response
#print(name)
#j_printer = pprint.PrettyPrinter(indent=2)
@@ -48,70 +45,63 @@ def getJsonInfo(mgrURL, reqURL):
#for record in json_response[key]:
# j_printer.pprint(record)
def removeEndps(mgrURL, endpNames):
for name in endpNames:
#print(f"Removing endp {name}")
data = {
"endp_name":name
}
jsonReq(mgrURL, "cli-json/rm_endp", data)
def removeCX(mgrURL, cxNames):
for name in cxNames:
#print(f"Removing CX {name}")
data = {
"test_mgr":"all",
"cx_name":name
}
jsonReq(mgrURL,"cli-json/rm_cx", data)
print("Checking for LANforge Client")
response = getJsonInfo(mgrURL, 'port/1/1/wiphy0')
timeout = 0
while response == None and timeout != 300:
duration = 0
while ((response == None) and (duration < 300)):
print("LANforge Client not found sleeping 5 seconds")
timeout += 5
time.sleep(5)
duration += 2
time.sleep(2)
response = getJsonInfo(mgrURL, 'port/1/1/wiphy0')
#print(response)
if timeout == 300:
if duration >= 300:
print("Could not connect to LANforge Client")
sys.exit(1)
print("See home/lanforge/Documents/connectTestLogs/connectTestLatest for specific values on latest test")
#Create stations and turn dhcp on
print("Creating station and turning on dhcp")
url = "cli-json/add_sta"
url = "port/1/1/sta00000"
debugOn = True
response = getJsonInfo(mgrURL, url)
if (response is not None):
if (response["interface"] is not None):
print("removing old station")
LFUtils.removePortByName("1.1.sta00000", mgrURL)
time.sleep(1)
url = "cli-json/add_sta"
data = {
"shelf":1,
"resource":1,
"radio":"wiphy0",
"sta_name":"sta00000",
"ssid":"jedway-wpa2-x2048-5-1",
"key":"jedway-wpa2-x2048-5-1",
"mode":1,
"mac":"xx:xx:xx:xx:*:xx",
"flags":1024 #0x400 | 1024
"shelf":1,
"resource":1,
"radio":"wiphy0",
"sta_name":"sta00000",
"ssid":"jedway-wpa2-x2048-5-1",
"key":"jedway-wpa2-x2048-5-1",
"mode":1,
"mac":"xx:xx:xx:xx:*:xx",
"flags":1024 #0x400 | 1024
}
print("adding new station")
jsonReq(mgrURL, url, data)
time.sleep(5)
time.sleep(1)
reqURL = "cli-json/set_port"
data = {
"shelf":1,
"resource":1,
"port":"sta00000",
"current_flags": 2147483648, #0x80000000 | 2147483648
"interest":16386 # 0x4002 | 16386
"shelf":1,
"resource":1,
"port":"sta00000",
"current_flags": 2147483648, #0x80000000 | 2147483648
"interest":16386 # 0x4002 | 16386
}
jsonReq(mgrURL,reqURL,data)
print("configuring port")
jsonReq(mgrURL, reqURL, data)
time.sleep(10)
time.sleep(5)
eth1IP = getJsonInfo(mgrURL, "port/1/1/eth1")
if eth1IP['interface']['ip'] == "0.0.0.0":
@@ -122,27 +112,26 @@ data = { "shelf":1,
"resource":1,
"port":"sta0000",
"probe_flags":1 }
jsonReq(mgrURL,reqURL,data)
jsonReq(mgrURL, reqURL, data)
staIP = getJsonInfo(mgrURL, "port/1/1/sta00000")
timeout = 0
station_info = getJsonInfo(mgrURL, "port/1/1/sta00000?fields=port,ip")
duration = 0
maxTime = 300
while staIP['interface']['ip'] == "0.0.0.0" and timeout != maxTime:
ip = "0.0.0.0"
while ((ip == "0.0.0.0") and (duration < maxTime)):
print("Station failed to get IP. Waiting 10 seconds...")
staIP = getJsonInfo(mgrURL, "port/1/1/sta00000")
timeout += 10
time.sleep(10)
if timeout == maxTime:
station_info = getJsonInfo(mgrURL, "port/1/1/sta00000?fields=port,ip")
#LFUtils.debug_printer.pprint(station_info)
if ((station_info is not None) and ("interface" in station_info) and ("ip" in station_info["interface"])):
ip = station_info["interface"]["ip"]
duration += 2
time.sleep(2)
if duration >= maxTime:
print("sta00000 failed to get an ip. Ending test")
print("Cleaning up...")
reqURL = "cli-json/rm_vlan"
data = {
"shelf":1,
"resource":1,
"port":"sta00000"
}
jsonReq(mgrURL, reqURL, data)
removePortByName("1.sta00000", mgrURL)
sys.exit(1)
@@ -160,14 +149,14 @@ time.sleep(.5)
#create l4 endpoint
url = "cli-json/add_l4_endp"
data = {
"alias":"l4Test",
"shelf":1,
"resource":1,
"port":"sta00000",
"type":"l4_generic",
"timeout":1000,
"url_rate":600,
"url":"dl http://10.40.0.1/ /dev/null"
"alias":"l4Test",
"shelf":1,
"resource":1,
"port":"sta00000",
"type":"l4_generic",
"timeout":1000,
"url_rate":600,
"url":"dl http://10.40.0.1/ /dev/null"
}
jsonReq(mgrURL, url, data)
@@ -176,10 +165,10 @@ time.sleep(.5)
#create cx for l4_endp
url = "cli-json/add_cx"
data = {
"alias":"CX_l4Test",
"test_mgr":"default_tm",
"tx_endp":"l4Test",
"rx_endp":"NA"
"alias":"CX_l4Test",
"test_mgr":"default_tm",
"tx_endp":"l4Test",
"rx_endp":"NA"
}
jsonReq(mgrURL, url, data)
@@ -188,12 +177,12 @@ time.sleep(.5)
#create fileio endpoint
url = "cli-json/add_file_endp"
data = {
"alias":"fioTest",
"shelf":1,
"resource":1,
"port":"sta00000",
"type":"fe_nfs",
"directory":"/mnt/fe-test"
"alias":"fioTest",
"shelf":1,
"resource":1,
"port":"sta00000",
"type":"fe_nfs",
"directory":"/mnt/fe-test"
}
jsonReq(mgrURL,url,data)
@@ -202,10 +191,10 @@ time.sleep(.5)
#create fileio cx
url = "cli-json/add_cx"
data = {
"alias":"CX_fioTest",
"test_mgr":"default_tm",
"tx_endp":"fioTest",
"rx_endp":"NA"
"alias":"CX_fioTest",
"test_mgr":"default_tm",
"tx_endp":"fioTest",
"rx_endp":"NA"
}
jsonReq(mgrURL,url,data)
@@ -223,10 +212,10 @@ time.sleep(.5)
#create generic cx
url = "cli-json/add_cx"
data = {
"alias":"CX_genTest1",
"test_mgr":"default_tm",
"tx_endp":"genTest1",
"rx_endp":"genTest2"
"alias":"CX_genTest1",
"test_mgr":"default_tm",
"tx_endp":"genTest1",
"rx_endp":"genTest2"
}
jsonReq(mgrURL,url,data)
@@ -235,20 +224,20 @@ time.sleep(.5)
#create redirects for wanlink
url = "cli-json/add_rdd"
data = {
"shelf":1,
"resource":1,
"port":"rdd0",
"peer_ifname":"rdd1"
"shelf":1,
"resource":1,
"port":"rdd0",
"peer_ifname":"rdd1"
}
jsonReq(mgrURL,url,data)
url = "cli-json/add_rdd"
data = {
"shelf":1,
"resource":1,
"port":"rdd1",
"peer_ifname":"rdd0"
"shelf":1,
"resource":1,
"port":"rdd1",
"peer_ifname":"rdd0"
}
jsonReq(mgrURL,url,data)
@@ -257,18 +246,18 @@ time.sleep(.5)
#reset redirect ports
url = "cli-json/reset_port"
data = {
"shelf":1,
"resource":1,
"port":"rdd0"
"shelf":1,
"resource":1,
"port":"rdd0"
}
jsonReq(mgrURL,url,data)
url = "cli-json/reset_port"
data = {
"shelf":1,
"resource":1,
"port":"rdd1"
"shelf":1,
"resource":1,
"port":"rdd1"
}
jsonReq(mgrURL,url,data)
@@ -278,24 +267,24 @@ time.sleep(.5)
#create wanlink endpoints
url = "cli-json/add_wl_endp"
data = {
"alias":"wlTest1",
"shelf":1,
"resource":1,
"port":"rdd0",
"latency":20,
"max_rate":1544000
"alias":"wlTest1",
"shelf":1,
"resource":1,
"port":"rdd0",
"latency":20,
"max_rate":1544000
}
jsonReq(mgrURL,url,data)
url = "cli-json/add_wl_endp"
data = {
"alias":"wlTest2",
"shelf":1,
"resource":1,
"port":"rdd1",
"latency":30,
"max_rate":1544000
"alias":"wlTest2",
"shelf":1,
"resource":1,
"port":"rdd1",
"latency":30,
"max_rate":1544000
}
jsonReq(mgrURL,url,data)
@@ -304,10 +293,10 @@ time.sleep(.5)
#create wanlink cx
url = "cli-json/add_cx"
data = {
"alias":"CX_wlTest1",
"test_mgr":"default_tm",
"tx_endp":"wlTest1",
"rx_endp":"wlTest2"
"alias":"CX_wlTest1",
"test_mgr":"default_tm",
"tx_endp":"wlTest1",
"rx_endp":"wlTest2"
}
jsonReq(mgrURL,url,data)
@@ -349,14 +338,7 @@ except Exception as e:
print("Something went wrong")
print(e)
print("Cleaning up...")
reqURL = "cli-json/rm_vlan"
data = {
"shelf":1,
"resource":1,
"port":"sta00000"
}
jsonReq(mgrURL, reqURL, data)
LFUtils.removePortByName("1.sta00000", mgrURL)
endpNames = ["testTCP-A", "testTCP-B",
"testUDP-A", "testUDP-B",
@@ -522,14 +504,7 @@ print("\n")
#remove all endpoints and cxs
print("Cleaning up...")
reqURL = "cli-json/rm_vlan"
data = {
"shelf":1,
"resource":1,
"port":"sta00000"
}
jsonReq(mgrURL, reqURL, data)
LFUtils.removePortByName("1.sta00000", mgrURL)
endpNames = ["testTCP-A", "testTCP-B",
"testUDP-A", "testUDP-B",

View File

@@ -28,6 +28,13 @@ import java.net.URL;
import java.util.*;
import java.nio.file.*;
/** Process a set of test results generated by the CI/CD process for a test type in a single testbed.
* There are currently no automated comparisons done across different testbeds.
* Generate historical graphs and links to each test run.
* Each test results directory would be packaged up by the lf_gui_report_summary.pl script,
* called by the basic_regression.bash automated regression test script.
* Example: java kpi --dir /var/www/html/tip/testbeds/ferndale-basic-01/reports/basic
*/
public class kpi {
String lc_osname;
String home_dir;
@@ -52,6 +59,8 @@ public class kpi {
public static String HTML_COLOR_FAIL = "#f9c5c0";
public static String HTML_COLOR_WARNING = "#f8f6ad";
public static String PF = "PASS-FAIL";
public kpi() {
priv_init();
}
@@ -124,6 +133,9 @@ public class kpi {
Vector<String> test_namesv = new Vector();
Vector<Run> runs = new Vector();
test_names.put(PF, PF);
test_namesv.add(PF);
try {
DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(dir));
for (Path file: stream) {
@@ -220,6 +232,28 @@ public class kpi {
runs.sort(new SortbyDate());
// Create meta-test results for each run.
for (int i = 0; i<runs.size(); i++) {
Run run = runs.elementAt(i);
Test t = new Test(PF);
run.addTest(t);
// Build fake kpi title and add that,
// then add fake kpi results line
// Example lines from a kpi.csv
//Date test-rig dut-hw-version dut-sw-version dut-model-num dut-serial-num test-priority test-id short-description pass/fail numeric-score test details Units Graph-Group Subtest-Pass Subtest-Fail
//1590190791848 Ferndale-01-Basic Linksys-EA8300 d88ea40 Linksys-EA8300 90 AP Auto Basic Client Connectivity Dual-Band MIN 8.0 Minimum CX time (ms) Time (ms) CX-Time 0 0
t.addLine("Date test-rig dut-hw-version dut-sw-version dut-model-num dut-serial-num test-priority test-id short-description pass/fail numeric-score test details Units Graph-Group Subtest-Pass Subtest-Fail");
t.addLine(run.getDateMs() + kpi.in_sep + run.getTestRig() + kpi.in_sep + run.getDutHwVer() + kpi.in_sep + run.getDutSwVer()
+ kpi.in_sep + run.getDutModelNum() + kpi.in_sep + run.getDutSerNum() + kpi.in_sep + "100" + kpi.in_sep + PF
+ kpi.in_sep + "Total Pass" + kpi.in_sep + kpi.in_sep + run.getPass() + kpi.in_sep + "Total passing tests and sub-tests for this run."
+ kpi.in_sep + "Number" + kpi.in_sep + PF + kpi.in_sep + "0" + kpi.in_sep + "0");
t.addLine(run.getDateMs() + kpi.in_sep + run.getTestRig() + kpi.in_sep + run.getDutHwVer() + kpi.in_sep + run.getDutSwVer()
+ kpi.in_sep + run.getDutModelNum() + kpi.in_sep + run.getDutSerNum() + kpi.in_sep + "100" + kpi.in_sep + PF
+ kpi.in_sep + "Total Failures" + kpi.in_sep + kpi.in_sep + run.getFail() + kpi.in_sep + "Total failing tests and sub-tests for this run."
+ kpi.in_sep + "Number" + kpi.in_sep + PF + kpi.in_sep + "0" + kpi.in_sep + "0");
}
// Link to latest test run that has the test id
Hashtable<String, Run> test_id_links = new Hashtable();
@@ -344,8 +378,10 @@ public class kpi {
String hk_str = hk;
Run last = test_id_links.get(hk);
if (last != null) {
hk_str = "<a href=\"" + last.getName() + "/" + hk + "/index.html\">" + hk + "</a>";
if (!hk.equals(PF)) {
if (last != null) {
hk_str = "<a href=\"" + last.getName() + "/" + hk + "/index.html\">" + hk + "</a>";
}
}
StringBuffer change = new StringBuffer();
@@ -359,7 +395,7 @@ public class kpi {
}
}
String row_str = ("<tr><td>" + hk_str + "</td><td>" + title + "</td><td><a href=\"" + npng + "\"><img src=\"" + npngt + "\"></a></td><td>"
String row_str = ("<tr><td>" + hk_str + "</td><td>" + title + "</td><td><a href=\"./" + npng + "\"><img src=\"./" + npngt + "\"></a></td><td>"
+ change + "</td></tr>\n");
if (csv.getPriority() >= 100) {
scores.append(row_str);
@@ -462,11 +498,13 @@ public class kpi {
String hk_str = hk;
Run last = test_id_links.get(hk);
if (last != null) {
hk_str = "<a href=\"" + last.getName() + "/" + hk + "/index.html\">" + hk + "</a>";
if (!hk.equals(PF)) {
if (last != null) {
hk_str = "<a href=\"" + last.getName() + "/" + hk + "/index.html\">" + hk + "</a>";
}
}
groups.append("<tr><td>" + hk_str + "</td><td>" + title + "</td><td><a href=\"" + npng + "\"><img src=\"" + npngt + "\"></a></td></tr>\n");
groups.append("<tr><td>" + hk_str + "</td><td>" + title + "</td><td><a href=\"./" + npng + "\"><img src=\"./" + npngt + "\"></a></td></tr>\n");
}
catch (Exception ee) {
@@ -607,6 +645,12 @@ public class kpi {
}
}
/** This holds a datapoint for a particular test result for each of the Runs.
* This is used to generate historical graphs and comparisons.
* If a test generates two KPI results, there will be two HistRows for that
* test case.
*/
class HistRow {
String fname;
String name;
@@ -654,6 +698,9 @@ class HistRow {
}
}
/** This holds all HistRow objects for each test name for each run.
* It is used for generating historical graphs and comparisons.
*/
class History {
String name;
Vector<HistRow> csv = new Vector();
@@ -678,6 +725,9 @@ class History {
}
}
/** A Row represents a single KPI csv data point. The csv is split into tokens
* for easier manipulation by other code.
*/
class Row {
Vector<String> rdata = new Vector();
String short_desc_key = null;
@@ -759,6 +809,11 @@ class Row {
}
}
/** A test contains information on one executation of one test. For instance,
* a wifi-capacity-test would be a single test. It may create a KPI.csv file with
* multiple rows. It may also have logs for this test run, and KPI graph images.
* A Run consists of multiple tests.
*/
class Test {
String name;
Vector<String> titles = null;
@@ -970,6 +1025,10 @@ class Test {
}//Test
/** A Run is a collection of tests. This class encompasses the entire regression test
* for a particular flavor of test (basic, fast, etc).
* The CI/CD process will create one or more Runs per build artifact per testbed.
*/
class Run {
String name;
Hashtable<String, Test> tests = new Hashtable();
@@ -1070,6 +1129,13 @@ class Run {
return "";
}
String getDutSerNum() {
Test t = getFirstTest();
if (t != null)
return t.getDutSerialNum();
return "";
}
String getDutModelNum() {
Test t = getFirstTest();
if (t != null)

View File

@@ -127,13 +127,13 @@ class LFRequest:
if (show_error):
print("----- get() HTTPError: --------------------------------------------")
print("<%s> HTTP %s: %s"%(myrequest.get_full_url(), error.code, error.reason, ))
print("Error: ", sys.exc_info()[0])
print("Request URL:", myrequest.get_full_url())
print("Request Content-type:", myrequest.get_header('Content-type'))
print("Request Accept:", myrequest.get_header('Accept'))
print("Request Data:")
LFUtils.debug_printer.pprint(myrequest.data)
if (error.code != 404):
print("Error: ", sys.exc_info()[0])
print("Request URL:", myrequest.get_full_url())
print("Request Content-type:", myrequest.get_header('Content-type'))
print("Request Accept:", myrequest.get_header('Accept'))
print("Request Data:")
LFUtils.debug_printer.pprint(myrequest.data)
if (error.headers):
# the HTTPError is of type HTTPMessage a subclass of email.message
@@ -143,7 +143,7 @@ class LFRequest:
if (len(myresponses) > 0):
print("----- Response: --------------------------------------------------------")
LFUtils.debug_printer.pprint(responses[0].reason)
LFUtils.debug_printer.pprint(myresponses[0].reason)
print("------------------------------------------------------------------------")
except urllib.error.URLError as uerror:
if (show_error):

View File

@@ -2,6 +2,7 @@
# Define useful common methods -
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
import sys
import os
if sys.version_info[0] != 3:
print("This script requires Python 3")
exit()
@@ -255,7 +256,7 @@ def waitUntilPortsAdminDown(resource_id=1, base_url="http://localhost:8080", por
continue
if "interface" in json_response:
json_response = json_response['interface']
if json_response['down'] is "false":
if json_response['down'] == "false":
up_stations.append(port_name)
sleep(1)
return None
@@ -275,7 +276,7 @@ def waitUntilPortsAdminUp(resource_id=1, base_url="http://localhost:8080", port_
continue
if "interface" in json_response:
json_response = json_response['interface']
if json_response['down'] is "true":
if json_response['down'] == "true":
down_stations.append(port_name)
sleep(1)
return None
@@ -313,9 +314,66 @@ def waitUntilPortsAppear(resource_id=1, base_url="http://localhost:8080", port_l
else:
lf_r = LFRequest.LFRequest(base_url+"/cli-form/nc_show_ports")
lf_r.addPostData({"shelf":1, "resource":resource_id, "port":port_name, "flags":1})
lr_r.formPost()
lf_r.formPost()
sleep(2)
print("These stations appeared: "+", ".join(found_stations))
return None
###
def removePort(resource, port_name, baseurl="http://localhost:8080/"):
lf_r = LFRequest.LFRequest(baseurl+"cli-json/rm_vlan")
lf_r.addPostData({
"shelf": 1,
"resource": resource,
"port": port_name
})
lf_r.jsonPost(False)
def removePortByName(port_name, baseurl="http://localhost:8080/"):
if ((port_name is None) or (port_name == "")):
print("No port name")
return;
if (port_name.index(".") < 0):
print("removePortByName: Please use short EID port names like: 2.sta1")
return
resource = port_name[0 : port_name.index(".")]
name = port_name[port_name.index(".")+1 : ]
if (name.index(".") >= 0):
name = name[name.index(".")+1 : ]
lf_r = LFRequest.LFRequest(baseurl+"cli-json/rm_vlan")
lf_r.addPostData({
"shelf": 1,
"resource": resource,
"port": name
})
lf_r.jsonPost(True)
def removeCX(mgrURL, cxNames):
for name in cxNames:
#print(f"Removing CX {name}")
data = {
"test_mgr":"all",
"cx_name":name
}
lf_r = LFRequest.LFRequest(mgrURL+"cli-json/rm_cx")
lf_r.addPostData(data)
lf_r.jsonPost()
def removeEndps(mgrURL, endpNames):
for name in endpNames:
#print(f"Removing endp {name}")
data = {
"endp_name":name
}
lf_r = LFRequest.LFRequest(mgrURL+"cli-json/rm_endp")
lf_r.addPostData(data)
lf_r.jsonPost()
def execWrap(cmd):
if os.system(cmd) != 0:
print("\nError with " + cmd + ",bye\n")
exit(1)
###

View File

@@ -1,21 +1,29 @@
#!/bin/bash
export DISPLAY=:1
HL="/home/lanforge"
HLD="${HL}/Documents"
scripts="${HLD}/lanforge-scripts"
GUILog="/home/lanforge/Documents/GUILog.txt"
GUIUpdate="/home/lanforge/Documents/GUIUpdateLog.txt"
CTLGUI="/home/lanforge/Documents/connectTestGUILog.txt"
CTLH="/home/lanforge/Documents/connectTestHLog.txt"
verNum="5.4.2"
python3 /home/lanforge/Documents/lanforge-scripts/auto-install-gui.py --versionNumber $verNum &> $GUIUpdate
for f in $GUILog $GUIUpdate $CTLGUI $CTLH; do rm -f $f; done
python3 ${scripts}/auto-install-gui.py --versionNumber $verNum &> $GUIUpdate
sleep 5
grep -q "Current GUI version up to date" $GUIUpdate && exit
python3 /home/lanforge/Documents/lanforge-scripts/connectTest.py &> $CTLGUI
python3 ${scripts}/connectTest.py &> $CTLGUI
sleep 1
pgrep java | xargs kill
sleep 1
/home/lanforge/LANforgeGUI_5.4.2/lfclient.bash -daemon -s localhost &> $GUILog &
python3 /home/lanforge/Documents/lanforge-scripts/connectTest.py &> $CTLH
${HL}/LANforgeGUI_5.4.2/lfclient.bash -daemon -s localhost &> $GUILog &
sleep 5
python3 ${scripts}/connectTest.py &> $CTLH
sleep 1
echo "Logs Attached" | mail -s 'GUI Update Logs' -a $GUILog -a $GUIUpdate -a $CTLGUI -a $CTLH test.notice@candelatech.com
echo "Logs Attached" | mail -s 'GUI Update Logs' -a $GUILog -a $GUIUpdate -a $CTLGUI -a $CTLH "test.notice@candelatech.com"