From fa96eda548a7eb642d08cb05a4387aec0184ecb5 Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Wed, 8 Jan 2025 16:46:13 -0500 Subject: [PATCH 1/6] save --- deploy/docker-compose-new/deploy.sh | 58 ++++++++++---------- deploy/docker-compose-new/docker-compose.yml | 33 ++++++++++- 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/deploy/docker-compose-new/deploy.sh b/deploy/docker-compose-new/deploy.sh index 63f6e33..4b8f7e3 100755 --- a/deploy/docker-compose-new/deploy.sh +++ b/deploy/docker-compose-new/deploy.sh @@ -4,10 +4,10 @@ DIR=$( cd "$(dirname "$0")" || exit; pwd -P ) ROOT_DIR=$DIR/../../ pushd "$ROOT_DIR" || exit -echo "Building Simulator..." -if [[ "$(docker images -q xmidt/simulator:latest 2> /dev/null)" == "" ]]; then - docker build -t xmidt/simulator:latest $ROOT_DIR/simulator -fi +# echo "Building Simulator..." +# if [[ "$(docker images -q xmidt/simulator:latest 2> /dev/null)" == "" ]]; then +# docker build -t xmidt/simulator:latest $ROOT_DIR/simulator +# fi popd echo "Running services..." @@ -25,29 +25,29 @@ if [[ $? -ne 0 ]]; then exit 1 fi -sleep 10 -AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb --endpoint-url http://localhost:8000 describe-table --table-name gifnoc --region us-east-2 --output text > /dev/null 2> /dev/null -if [[ $? -ne 0 ]]; then - AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb --endpoint-url http://localhost:8000 create-table \ - --table-name gifnoc \ - --attribute-definitions \ - AttributeName=bucket,AttributeType=S \ - AttributeName=expires,AttributeType=N \ - AttributeName=id,AttributeType=S \ - --key-schema \ - AttributeName=bucket,KeyType=HASH \ - AttributeName=id,KeyType=RANGE \ - --global-secondary-indexes \ - "[{\"IndexName\": \"Expires-index\",\"KeySchema\":[{\"AttributeName\":\"bucket\",\"KeyType\":\"HASH\"}, {\"AttributeName\":\"expires\",\"KeyType\":\"RANGE\"}], \ - \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 10, \"WriteCapacityUnits\": 5 },\"Projection\":{\"ProjectionType\":\"ALL\"}}]" \ - --provisioned-throughput \ - ReadCapacityUnits=10,WriteCapacityUnits=5 \ - --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES \ - --region us-east-2 \ - --output text +# sleep 10 +# AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb --endpoint-url http://localhost:8000 describe-table --table-name gifnoc --region us-east-2 --output text > /dev/null 2> /dev/null +# if [[ $? -ne 0 ]]; then +# AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb --endpoint-url http://localhost:8000 create-table \ +# --table-name gifnoc \ +# --attribute-definitions \ +# AttributeName=bucket,AttributeType=S \ +# AttributeName=expires,AttributeType=N \ +# AttributeName=id,AttributeType=S \ +# --key-schema \ +# AttributeName=bucket,KeyType=HASH \ +# AttributeName=id,KeyType=RANGE \ +# --global-secondary-indexes \ +# "[{\"IndexName\": \"Expires-index\",\"KeySchema\":[{\"AttributeName\":\"bucket\",\"KeyType\":\"HASH\"}, {\"AttributeName\":\"expires\",\"KeyType\":\"RANGE\"}], \ +# \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 10, \"WriteCapacityUnits\": 5 },\"Projection\":{\"ProjectionType\":\"ALL\"}}]" \ +# --provisioned-throughput \ +# ReadCapacityUnits=10,WriteCapacityUnits=5 \ +# --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES \ +# --region us-east-2 \ +# --output text - AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb \ - --endpoint-url http://localhost:8000 --region us-east-2 update-time-to-live \ - --table-name gifnoc --time-to-live-specification "Enabled=true, AttributeName=expires" \ - --output text -fi +# AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb \ +# --endpoint-url http://localhost:8000 --region us-east-2 update-time-to-live \ +# --table-name gifnoc --time-to-live-specification "Enabled=true, AttributeName=expires" \ +# --output text +# fi diff --git a/deploy/docker-compose-new/docker-compose.yml b/deploy/docker-compose-new/docker-compose.yml index 0f18fc5..78f97b8 100644 --- a/deploy/docker-compose-new/docker-compose.yml +++ b/deploy/docker-compose-new/docker-compose.yml @@ -7,7 +7,7 @@ services: depends_on: - dynamodb environment: - - "AWS_ENDPOINT=http://dynamodb:8000" + - "AWS_ENDPOINT=http://dynamodb:4566" - "TRACING_PROVIDER_NAME=zipkin" - "TRACING_PROVIDER_ENDPOINT=http://zipkin:9411/api/v2/spans" - "ACCESS_LEVEL_CAPABILITY=x1:issuer:test:.*:all" @@ -65,7 +65,14 @@ services: networks: - xmidt ports: - - "9000:9000" + - "4566:4566" + healthcheck: + test: + ["CMD-SHELL",'if [ "$(curl -s -o /dev/null -I -w ''%{http_code}'' http://localhost:4566)" == "400" ]; then exit 0; else exit 1; fi',] + interval: 5s + timeout: 30s + retries: 3 + start_period: 5s petasos: container_name: petasos depends_on: @@ -187,6 +194,28 @@ services: - xmidt ports: - "6100-6103:6100-6103" + awscli-dynamo: + image: amazon/aws-cli + networks: + - xmidt + depends_on: + dynamodb: + condition: service_healthy + ports: + - "4577:4577" + environment: + AWS_ACCESS_KEY_ID: accessKey + AWS_SECRET_ACCESS_KEY: secretKey + AWS_REGION: local + command: "dynamodb --endpoint-url http://dynamodb:4566 create-table \ + --table-name gifnoc \ + --attribute-definitions \ + AttributeName=bucket,AttributeType=S \ + AttributeName=expires,AttributeType=N \ + AttributeName=id,AttributeType=S \ + --key-schema \ + AttributeName=bucket,KeyType=HASH \ + AttributeName=id,KeyType=RANGE" zipkin: image: openzipkin/zipkin networks: From 3019416d25e582420c40d70533047feedfc5beae Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Wed, 8 Jan 2025 16:55:47 -0500 Subject: [PATCH 2/6] save --- deploy/docker-compose-new/deploy.sh | 12 ++++++------ deploy/docker-compose-new/docker-compose.yml | 16 ++++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/deploy/docker-compose-new/deploy.sh b/deploy/docker-compose-new/deploy.sh index 4b8f7e3..f9eb6a4 100755 --- a/deploy/docker-compose-new/deploy.sh +++ b/deploy/docker-compose-new/deploy.sh @@ -3,12 +3,12 @@ DIR=$( cd "$(dirname "$0")" || exit; pwd -P ) ROOT_DIR=$DIR/../../ -pushd "$ROOT_DIR" || exit -# echo "Building Simulator..." -# if [[ "$(docker images -q xmidt/simulator:latest 2> /dev/null)" == "" ]]; then -# docker build -t xmidt/simulator:latest $ROOT_DIR/simulator -# fi -popd +# pushd "$ROOT_DIR" || exit +# # echo "Building Simulator..." +# # if [[ "$(docker images -q xmidt/simulator:latest 2> /dev/null)" == "" ]]; then +# # docker build -t xmidt/simulator:latest $ROOT_DIR/simulator +# # fi +# popd echo "Running services..." CONSUL_VERSION=${CONSUL_VERSION:-1.12.6} \ diff --git a/deploy/docker-compose-new/docker-compose.yml b/deploy/docker-compose-new/docker-compose.yml index 78f97b8..c46e4eb 100644 --- a/deploy/docker-compose-new/docker-compose.yml +++ b/deploy/docker-compose-new/docker-compose.yml @@ -62,17 +62,22 @@ services: container_name: dynamo-xmidt hostname: dynamodb image: amazon/dynamodb-local + environment: + AWS_ACCESS_KEY_ID: accessKey + AWS_SECRET_ACCESS_KEY: secretKey + AWS_REGION: local + command: -jar DynamoDBLocal.jar -inMemory -sharedDb -port 4566 networks: - xmidt ports: - "4566:4566" healthcheck: - test: + test: ["CMD-SHELL",'if [ "$(curl -s -o /dev/null -I -w ''%{http_code}'' http://localhost:4566)" == "400" ]; then exit 0; else exit 1; fi',] - interval: 5s - timeout: 30s - retries: 3 - start_period: 5s + interval: 5s + timeout: 30s + retries: 3 + start_period: 5s petasos: container_name: petasos depends_on: @@ -222,4 +227,3 @@ services: - xmidt ports: - "9411:9411" -version: "3.8" From 2a7f8bd3136faca3b5c511aea6e568541660d306 Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Wed, 8 Jan 2025 17:31:05 -0500 Subject: [PATCH 3/6] save --- deploy/docker-compose-new/docFiles/xmidt-agent.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/docker-compose-new/docFiles/xmidt-agent.yaml b/deploy/docker-compose-new/docFiles/xmidt-agent.yaml index 1f21bff..03e3c84 100644 --- a/deploy/docker-compose-new/docFiles/xmidt-agent.yaml +++ b/deploy/docker-compose-new/docFiles/xmidt-agent.yaml @@ -31,3 +31,6 @@ mock_tr_181: file_path: /mock_tr181.json lib_parodus: parodus_service_url: "tcp://127.0.0.1:6666" +logger: + level: debug + development: true From 3b5f4d350f05599e3ec39ed50601ac366a34eb5e Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Tue, 21 Jan 2025 12:09:40 -0500 Subject: [PATCH 4/6] remove commented code --- deploy/docker-compose-new/deploy.sh | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/deploy/docker-compose-new/deploy.sh b/deploy/docker-compose-new/deploy.sh index f9eb6a4..03c719a 100755 --- a/deploy/docker-compose-new/deploy.sh +++ b/deploy/docker-compose-new/deploy.sh @@ -23,31 +23,4 @@ SIMULATOR_VERSION=${SIMULATOR_VERSION:-latest-amd64} \ docker compose -f $ROOT_DIR/deploy/docker-compose-new/docker-compose.yml up -d $@ if [[ $? -ne 0 ]]; then exit 1 -fi - -# sleep 10 -# AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb --endpoint-url http://localhost:8000 describe-table --table-name gifnoc --region us-east-2 --output text > /dev/null 2> /dev/null -# if [[ $? -ne 0 ]]; then -# AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb --endpoint-url http://localhost:8000 create-table \ -# --table-name gifnoc \ -# --attribute-definitions \ -# AttributeName=bucket,AttributeType=S \ -# AttributeName=expires,AttributeType=N \ -# AttributeName=id,AttributeType=S \ -# --key-schema \ -# AttributeName=bucket,KeyType=HASH \ -# AttributeName=id,KeyType=RANGE \ -# --global-secondary-indexes \ -# "[{\"IndexName\": \"Expires-index\",\"KeySchema\":[{\"AttributeName\":\"bucket\",\"KeyType\":\"HASH\"}, {\"AttributeName\":\"expires\",\"KeyType\":\"RANGE\"}], \ -# \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 10, \"WriteCapacityUnits\": 5 },\"Projection\":{\"ProjectionType\":\"ALL\"}}]" \ -# --provisioned-throughput \ -# ReadCapacityUnits=10,WriteCapacityUnits=5 \ -# --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES \ -# --region us-east-2 \ -# --output text - -# AWS_ACCESS_KEY_ID=accessKey AWS_SECRET_ACCESS_KEY=secretKey aws dynamodb \ -# --endpoint-url http://localhost:8000 --region us-east-2 update-time-to-live \ -# --table-name gifnoc --time-to-live-specification "Enabled=true, AttributeName=expires" \ -# --output text -# fi +fi \ No newline at end of file From b9af4ae3489d1cc05fd639b03e11731f93f86913 Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Tue, 21 Jan 2025 19:10:07 -0500 Subject: [PATCH 5/6] use a local themis.yaml config file --- .../docker-compose-new/docFiles/themis.yaml | 122 ++++++++++++++++++ deploy/docker-compose-new/docker-compose.yml | 2 + 2 files changed, 124 insertions(+) create mode 100644 deploy/docker-compose-new/docFiles/themis.yaml diff --git a/deploy/docker-compose-new/docFiles/themis.yaml b/deploy/docker-compose-new/docFiles/themis.yaml new file mode 100644 index 0000000..dde06d3 --- /dev/null +++ b/deploy/docker-compose-new/docFiles/themis.yaml @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: 2022 Comcast Cable Communications Management, LLC +# SPDX-License-Identifier: Apache-2.0 +--- +# The unique fully-qualified-domain-name of the server. It is provided to +# the X-Scytale-Server header for showing what server fulfilled the request +# sent. +# (Optional) +server: themis + +######################################## +# Labeling/Tracing via HTTP Headers Configuration +######################################## + +# Provides this build number to the X-Scytale-Build header for +# showing machine version information. The build number SHOULD +# match the scheme `version-build` but there is not a strict requirement. +# (Optional) +build: unknown + +# stage is used for doing complex spruce manipulation +# change this will only effect spruce and the vars referencing it +stage: "development" + +servers: + key: + address: :6500 + disableHTTPKeepAlives: true + header: + X-Midt-Server: + - themis + X-Midt-Version: + - unknown + issuer: + address: :6501 + disableHTTPKeepAlives: true + header: + X-Midt-Server: + - themis + X-Midt-Version: + - unknown + claims: + address: :6502 + disableHTTPKeepAlives: true + header: + X-Midt-Server: + - themis + X-Midt-Version: + - unknown + metrics: + address: :6503 + disableHTTPKeepAlives: true + + health: + address: :6504 + disableHTTPKeepAlives: true + header: + X-Midt-Server: + - themis + X-Midt-Version: + - unknown +health: + disableLogging: false + custom: + server: themis + +prometheus: + defaultNamespace: xmidt + defaultSubsystem: themis + +token: + alg: RS256 + nonce: true + notBeforeDelta: -15s + duration: 24h + claims: + - key: mac + header: X-Midt-Mac-Address + parameter: mac + - key: serial + header: X-Midt-Serial-Number + parameter: serial + - key: uuid + header: X-Midt-Uuid + parameter: uuid + - key: iss + value: "themis" + - key: trust + value: 1000 + - key: sub + value: "client-supplied" + - key: aud + value: "XMiDT" + - key: capabilities + value: + - x1:issuer:test:.*:all + metadata: + - key: mac + header: X-Midt-Mac-Address + parameter: mac + - key: serial + header: X-Midt-Serial-Number + parameter: serial + - key: uuid + header: X-Midt-Uuid + parameter: uuid + partnerID: + claim: partner-id + header: X-Midt-Partner-ID + parameter: pid + default: comcast + + key: + kid: docker + type: rsa + bits: 2048 + +log: + outputPaths: + - "stdout" + errorOutputPaths: + - "stdout" + level: debug diff --git a/deploy/docker-compose-new/docker-compose.yml b/deploy/docker-compose-new/docker-compose.yml index c46e4eb..0e5e640 100644 --- a/deploy/docker-compose-new/docker-compose.yml +++ b/deploy/docker-compose-new/docker-compose.yml @@ -184,6 +184,8 @@ services: - xmidt ports: - "6500-6504:6500-6504" + volumes: + - "./docFiles/themis.yaml:/etc/themis/themis.yaml" tr1d1um: container_name: tr1d1um depends_on: From 6b2b6a4f6a05cb785ad112cc791aea997fc2116a Mon Sep 17 00:00:00 2001 From: mpicci200_comcast Date: Wed, 22 Jan 2025 11:09:49 -0500 Subject: [PATCH 6/6] update readme --- deploy/docker-compose-new/README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/deploy/docker-compose-new/README.md b/deploy/docker-compose-new/README.md index d1efaa8..af8eab6 100644 --- a/deploy/docker-compose-new/README.md +++ b/deploy/docker-compose-new/README.md @@ -1,9 +1,5 @@ # Deploying XMiDT -## Docker/awscli - -In order to deploy into Docker, make sure [Docker is installed](https://docs.docker.com/install/). Additionally for goaws, make sure that awscli is intalled. - #### Deploy _**Note**_: While Tr1d1um is not part of XMiDT(it is WebPA), it is recommended to be brought up for current ease of use. Future releases will deprecate Tr1d1um. @@ -12,7 +8,9 @@ brought up for current ease of use. Future releases will deprecate Tr1d1um. 2. Run `deploy/docker-compose-new/deploy.sh` - This will build `goaws` locally. It will then run `docker-compose up` which uses images of `talaria`, `scytale`, `petasos`, `caduceus`, `xmidt-agent` (device simulator) and `tr1d1um` from dockerhub. + It will then run `docker-compose up` which uses images of `talaria`, `scytale`, `petasos`, `caduceus`, `xmidt-agent` (device simulator) and `tr1d1um` from dockerhub. + + It will also automatically build a dynamo db resource. To pull specific versions of the images, just set the `_VERSION` env variables when running the shell script. @@ -40,10 +38,10 @@ deal with just one datacenter. Since all ports are exposed, the names might seem #### Connection ##### Inside Docker -If the Parodus instance is inside of docker, life is easy! Just connect to the cluster with `petasos:6400`. +If the Xmidt-Agent instance is inside of docker, life is easy! Just connect to the cluster with `petasos:6400`. ##### Outside Docker -if the Parodus instance is outside of docker and the ports are exposed correctly, life +if the Xmidt-Agent instance is outside of docker and the ports are exposed correctly, life will be hard since you will need to handle the redirect. You can initially connect to 'localhost:6400' but on the redirect change `talaria-1:6210` to `localhost:6210`