feat: Read a list of optional extra equipement files

Signed-off-by: EstherLerouzic <esther.lerouzic@orange.com>
Change-Id: Ic521bbacd38b3bb60da3a364a069abfd1895d337

gnpy/example-data/extra_eqpt_config.json

@@ -0,0 +1,49 @@
+{
+  "Transceiver": [
+    {
+      "type_variety": "ZR400G",
+      "frequency": {
+        "min": 191.3e12,
+        "max": 196.1e12
+      },
+      "mode": [
+        {
+          "format": "SFF-ID:70",
+          "baud_rate": 60138546798,
+          "OSNR": 24,
+          "bit_rate": 400e9,
+          "roll_off": 0.2,
+          "tx_osnr": 34,
+          "min_spacing": 75e9,
+          "penalties": [
+            {
+              "chromatic_dispersion": 20e3,
+              "penalty_value": 0.5
+            },
+            {
+              "chromatic_dispersion": 0,
+              "penalty_value": 0
+            },
+            {
+              "pmd": 20,
+              "penalty_value": 0.5
+            },
+            {
+              "pdl": 1.5,
+              "penalty_value": 0
+            },
+            {
+              "pdl": 3.5,
+              "penalty_value": 1.8
+            },
+            {
+              "pdl": 3,
+              "penalty_value": 1.3
+            }
+          ],
+          "cost": 1
+        }
+      ]
+    }
+  ]
+}

gnpy/example-data/service_pluggable.json

@@ -0,0 +1,22 @@
+{
+  "path-request": [
+    {
+      "request-id": "0",
+      "source": "trx Brest_KLA",
+      "destination": "trx Lannion_CAS",
+      "src-tp-id": "trx Brest_KLA",
+      "dst-tp-id": "trx Lannion_CAS",
+      "bidirectional": false,
+      "path-constraints": {
+        "te-bandwidth": {
+          "technology": "flexi-grid",
+          "trx_type": "ZR400G",
+          "trx_mode": "SFF-ID:70",
+          "spacing": 100000000000.0,
+          "tx_power": 0.0015,
+          "path_bandwidth": 400000000000.0
+        }
+      }
+    }
+  ]
+}

gnpy/tools/cli_examples.py

@@ -12,6 +12,7 @@ import argparse
 import logging
 import sys
 from pathlib import Path
+from typing import List
 from math import ceil
 from numpy import mean
 
@@ -22,7 +23,7 @@ from gnpy.core.parameters import SimParams
 from gnpy.core.utils import lin2db, pretty_summary_print, per_label_average, watt2dbm
 from gnpy.topology.request import (ResultElement, jsontocsv, BLOCKING_NOPATH)
 from gnpy.tools.json_io import (load_equipment, load_network, load_json, load_requests, save_network,
-                                requests_from_json, save_json, load_initial_spectrum)
+                                requests_from_json, save_json, load_initial_spectrum, merge_equipment)
 from gnpy.tools.plots import plot_baseline, plot_results
 from gnpy.tools.worker_utils import designed_network, transmission_simulation, planning
 
@@ -43,11 +44,14 @@ def show_example_data_dir():
     print(f'{_examples_dir}/')
 
 
-def load_common_data(equipment_filename, topology_filename, simulation_filename, save_raw_network_filename):
+def load_common_data(equipment_filename: Path, extra_equipment_filenames: List[Path], topology_filename: Path,
-    """Load common configuration from JSON files"""
+                     simulation_filename: Path, save_raw_network_filename: Path):
+    """Load common configuration from JSON files, merging additional equipment if provided."""
 
     try:
         equipment = load_equipment(equipment_filename)
+        if extra_equipment_filenames:
+            merge_equipment(equipment, extra_equipment_filenames)
         network = load_network(topology_filename, equipment)
         if save_raw_network_filename is not None:
             save_network(network, save_raw_network_filename)
@@ -102,6 +106,10 @@ def _add_common_options(parser: argparse.ArgumentParser, network_default: Path):
     parser.add_argument('--no-insert-edfas', action='store_true',
                         help='Disable insertion of EDFAs after ROADMs and fibers '
                              'as well as splitting of fibers by auto-design.')
+    # Option for additional equipment files
+    parser.add_argument('-x', '--extra-equipment', nargs='+', type=Path,
+                        metavar=_help_fname_json, default=None,
+                        help='List of additional equipment files to complement the main equipment file.')
 
 
 def transmission_main_example(args=None):
@@ -125,7 +133,8 @@ def transmission_main_example(args=None):
     args = parser.parse_args(args if args is not None else sys.argv[1:])
     _setup_logging(args)
 
-    (equipment, network) = load_common_data(args.equipment, args.topology, args.sim_params, args.save_network_before_autodesign)
+    (equipment, network) = load_common_data(args.equipment, args.extra_equipment, args.topology, args.sim_params,
+                                            args.save_network_before_autodesign)
 
     if args.plot:
         plot_baseline(network)
@@ -313,7 +322,8 @@ def path_requests_run(args=None):
     _logger.info(f'Computing path requests {args.service_filename.name} into JSON format')
 
     (equipment, network) = \
-        load_common_data(args.equipment, args.topology, args.sim_params, args.save_network_before_autodesign)
+        load_common_data(args.equipment, args.extra_equipment, args.topology, args.sim_params,
+                         args.save_network_before_autodesign)
 
     # Build the network once using the default power defined in SI in eqpt config
 

gnpy/tools/json_io.py

@@ -368,6 +368,24 @@ def _spectrum_from_json(json_data: dict):
     return spectrum
 
 
+def merge_equipment(equipment: dict, additional_filenames: List[Path]):
+    """Merge additional equipment libraries into the base equipment dictionary.
+    Typical case is the use of third party transceivers which are not part of a the supplier library.
+
+    raise warnings if the same reference is used on two different libraries
+    """
+    for filename in additional_filenames:
+        extra_eqpt = load_equipment(filename)
+        # populate with default eqpt to streamline loading
+        for eqpt_type, extra_items in extra_eqpt.items():
+            for type_variety, item in extra_items.items():
+                if type_variety not in equipment[eqpt_type]:
+                    equipment[eqpt_type][type_variety] = item
+                else:
+                    msg = f'\n\tEquipment file {filename.name}: duplicate equipment entry found: {eqpt_type}-{type_variety}\n'
+                    _logger.warning(msg)
+
+
 def load_equipment(filename: Path) -> dict:
     """Load equipment, returns equipment dict
     """
@@ -388,6 +406,8 @@ def _update_dual_stage(equipment: dict) -> dict:
 
     Returns the updated equiment dictionary
     """
+    if 'Edfa' not in equipment:
+        return equipment
     edfa_dict = equipment['Edfa']
     for edfa in edfa_dict.values():
         if edfa.type_def == 'dual_stage':
@@ -409,6 +429,8 @@ def _update_dual_stage(equipment: dict) -> dict:
 def _update_band(equipment: dict) -> dict:
     """Creates a list of bands for this amplifier, and remove other parameters which are not applicable
     """
+    if 'Edfa' not in equipment:
+        return equipment
     amp_dict = equipment['Edfa']
     for amplifier in amp_dict.values():
         if amplifier.type_def != 'multi_band':
@@ -433,6 +455,8 @@ def _update_band(equipment: dict) -> dict:
 
 def _roadm_restrictions_sanity_check(equipment: dict):
     """verifies that booster and preamp restrictions specified in roadm equipment are listed in the edfa."""
+    if 'Roadm' not in equipment:
+        return equipment
     for roadm_type, roadm_eqpt in equipment['Roadm'].items():
         restrictions = roadm_eqpt.restrictions['booster_variety_list'] + \
             roadm_eqpt.restrictions['preamp_variety_list']
@@ -457,6 +481,19 @@ def _check_fiber_vs_raman_fiber(equipment: dict):
                                            f'disagrees for "{attr}": {a} != {b}')
 
 
+def _si_sanity_check(equipment):
+    """Check that 'default' key correctly exists in SI list. (There must be at list one element and it must be default)
+    If not create one entry in the list with this key.
+    """
+    if 'SI' not in equipment:
+        return
+    possible_SI = list(equipment['SI'].keys())
+    if 'default' not in possible_SI:
+        # Use "default" key in the equipment, using the first listed keys
+        equipment['SI']['default'] = equipment['SI'][possible_SI[0]]
+        del equipment['SI'][possible_SI[0]]
+
+
 def _equipment_from_json(json_data: dict, filename: Path) -> dict:
     """build global dictionnary eqpt_library that stores all eqpt characteristics:
     edfa type type_variety, fiber type_variety
@@ -494,11 +531,7 @@ def _equipment_from_json(json_data: dict, filename: Path) -> dict:
     equipment = _update_dual_stage(equipment)
     equipment = _update_band(equipment)
     _roadm_restrictions_sanity_check(equipment)
-    possible_SI = list(equipment['SI'].keys())
+    _si_sanity_check(equipment)
-    if 'default' not in possible_SI:
-        # Use "default" key in the equipment, using the first listed keys
-        equipment['SI']['default'] = equipment['SI'][possible_SI[0]]
-        del equipment['SI'][possible_SI[0]]
     return equipment
 
 

tests/data/extra_eqpt_config.json

@@ -0,0 +1,37 @@
+{
+  "Transceiver": [
+    {
+      "type_variety": "ZR400G",
+      "frequency": {
+        "min": 191.35e12,
+        "max": 196.1e12
+      },
+      "mode": [
+        {
+          "format": "400G",
+          "baud_rate": 60e9,
+          "OSNR": 24,
+          "bit_rate": 400e9,
+          "roll_off": 0.2,
+          "tx_osnr": 38,
+          "min_spacing": 75e9,
+          "cost": 1
+        }
+      ]
+    }
+  ],
+  "Edfa": [
+    {
+      "type_variety": "user_defined_default_amplifier",
+      "type_def": "variable_gain",
+      "gain_flatmax": 25,
+      "gain_min": 15,
+      "p_max": 21,
+      "nf_min": 6,
+      "nf_max": 10,
+      "advanced_config_from_json": "default_edfa_config.json",
+      "out_voa_auto": false,
+      "allowed_for_design": false
+    }
+  ]
+}

tests/invocation/logs_path_requests_run_extra_equipment

@@ -0,0 +1,33 @@
+INFO     gnpy.tools.cli_examples:cli_examples.py Computing path requests service_pluggable.json into JSON format
+WARNING  gnpy.tools.json_io:json_io.py 
+	WARNING missing type_variety attribute in eqpt_config.json[Roadm]
+	default value is type_variety = default
+
+WARNING  gnpy.tools.json_io:json_io.py 
+	Equipment file extra_eqpt_config.json: duplicate equipment entry found: Transceiver-ZR400G
+
+INFO     gnpy.tools.worker_utils:worker_utils.py List of disjunctions:
+[]
+INFO     gnpy.tools.worker_utils:worker_utils.py Aggregating similar requests
+INFO     gnpy.tools.worker_utils:worker_utils.py The following services have been requested:
+[PathRequest 0
+	source: 	trx Brest_KLA
+	destination:	trx Lannion_CAS
+	trx type:	ZR400G
+	trx mode:	SFF-ID:70
+	baud_rate:	60.13854679800001 Gbaud
+	bit_rate:	400.0 Gb/s
+	spacing:	100.0 GHz
+	power:  	0.0 dBm
+	tx_power_dbm:  	1.76 dBm
+	nb channels: 	48
+	path_bandwidth: 	400.0 Gbit/s
+	nodes-list:	[]
+	loose-list:	[]
+]
+INFO     gnpy.tools.worker_utils:worker_utils.py Propagating on selected path
+INFO     gnpy.topology.request:request.py 
+	request 0
+	Computing path from trx Brest_KLA to trx Lannion_CAS
+	with path constraint: ['trx Brest_KLA', 'trx Lannion_CAS']
+	Computed path (roadms):['roadm Brest_KLA', 'roadm Lannion_CAS']

tests/invocation/path_requests_run_extra_equipment

@@ -0,0 +1,22 @@
+List of disjunctions
+[]
+The following services have been requested:
+[PathRequest 0
+	source: 	trx Brest_KLA
+	destination:	trx Lannion_CAS
+	trx type:	ZR400G
+	trx mode:	SFF-ID:70
+	baud_rate:	60.13854679800001 Gbaud
+	bit_rate:	400.0 Gb/s
+	spacing:	100.0 GHz
+	power:  	0.0 dBm
+	tx_power_dbm:  	1.76 dBm
+	nb channels: 	48
+	path_bandwidth: 	400.0 Gbit/s
+	nodes-list:	[]
+	loose-list:	[]
+]
+Result summary
+req id   demand                              GSNR@bandwidth A-Z (Z-A)   GSNR@0.1nm A-Z (Z-A)        Receiver minOSNR               mode                    Gbit/s              nb of tsp pairs      N,M or blocking reason 
+0       trx Brest_KLA to trx Lannion_CAS :             21.3                    28.12                      26                   SFF-ID:70                  400.0                      1                   ([-280],[8])      
+Result summary shows mean GSNR and OSNR (average over all channels)

tests/test_invocation.py

@@ -18,11 +18,11 @@ SRC_ROOT = Path(__file__).parent.parent
     ('transmission_main_example__raman', None, transmission_main_example,
      ['gnpy/example-data/raman_edfa_example_network.json', '--sim', 'gnpy/example-data/sim_params.json', '--show-channels', ]),
     ('openroadm-v4-Stockholm-Gothenburg', None, transmission_main_example,
-     ['-e', 'gnpy/example-data/eqpt_config_openroadm_ver4.json', 'gnpy/example-data/Sweden_OpenROADMv4_example_network.json', ]),
+     ['gnpy/example-data/Sweden_OpenROADMv4_example_network.json', '-e', 'gnpy/example-data/eqpt_config_openroadm_ver4.json', ]),
     ('openroadm-v5-Stockholm-Gothenburg', None, transmission_main_example,
-     ['-e', 'gnpy/example-data/eqpt_config_openroadm_ver5.json', 'gnpy/example-data/Sweden_OpenROADMv5_example_network.json', ]),
+     ['gnpy/example-data/Sweden_OpenROADMv5_example_network.json', '-e', 'gnpy/example-data/eqpt_config_openroadm_ver5.json', ]),
     ('transmission_main_example_long', None, transmission_main_example,
-     ['-e', 'tests/data/eqpt_config.json', 'tests/data/test_long_network.json']),
+     ['tests/data/test_long_network.json', '-e', 'tests/data/eqpt_config.json']),
     ('spectrum1_transmission_main_example', None, transmission_main_example,
      ['--spectrum', 'gnpy/example-data/initial_spectrum1.json', 'gnpy/example-data/meshTopologyExampleV2.xls', ]),
     ('spectrum2_transmission_main_example', None, transmission_main_example,
@@ -32,14 +32,16 @@ SRC_ROOT = Path(__file__).parent.parent
     ('power_sweep_example', 'logs_power_sweep_example', transmission_main_example,
      ['tests/data/testTopology_expected.json', 'brest', 'rennes', '-e', 'tests/data/eqpt_config_sweep.json', '--pow', '3']),
     ('transmission_long_pow', None, transmission_main_example,
-     ['-e', 'tests/data/eqpt_config.json', 'tests/data/test_long_network.json', '--spectrum', 'gnpy/example-data/initial_spectrum2.json']),
+     ['tests/data/test_long_network.json', '-e', 'tests/data/eqpt_config.json', '--spectrum', 'gnpy/example-data/initial_spectrum2.json']),
     ('transmission_long_psd', None, transmission_main_example,
-     ['-e', 'tests/data/eqpt_config_psd.json', 'tests/data/test_long_network.json', '--spectrum', 'gnpy/example-data/initial_spectrum2.json', ]),
+     ['tests/data/test_long_network.json', '-e', 'tests/data/eqpt_config_psd.json', '--spectrum', 'gnpy/example-data/initial_spectrum2.json', ]),
     ('transmission_long_psw', None, transmission_main_example,
-     ['-e', 'tests/data/eqpt_config_psw.json', 'tests/data/test_long_network.json', '--spectrum', 'gnpy/example-data/initial_spectrum2.json', ]),
+     ['tests/data/test_long_network.json', '-e', 'tests/data/eqpt_config_psw.json', '--spectrum', 'gnpy/example-data/initial_spectrum2.json', ]),
     ('multiband_transmission', None, transmission_main_example,
      ['gnpy/example-data/multiband_example_network.json', 'Site_A', 'Site_D', '-e', 'gnpy/example-data/eqpt_config_multiband.json',
-      '--spectrum', 'gnpy/example-data/multiband_spectrum.json', '--show-channels'])
+      '--spectrum', 'gnpy/example-data/multiband_spectrum.json', '--show-channels']),
+    ('path_requests_run_extra_equipment', 'logs_path_requests_run_extra_equipment', path_requests_run,
+     ['gnpy/example-data/meshTopologyExampleV2.xls', 'gnpy/example-data/service_pluggable.json', '--extra-equipment', 'gnpy/example-data/extra_eqpt_config.json', 'tests/data/extra_eqpt_config.json'])
 ))
 def test_example_invocation(capfd, caplog, output, log, handler, args):
     """Make sure that our examples produce useful output"""