Deployment: enrichment store, values, quickstart install (#413)

* improvements for upload of files

* adding index file and namespace, and change install values

* adding index file and namespace, and change install values

* fix

* small fixes, shared zk and resources update

* small fix

* refactoring

Co-authored-by: Yassin Raman <yassin@gmail.co>
Co-authored-by: yasram1 <yasram1@github.com>

deployment/helm-k8s/resources/download.php

@@ -1,15 +1,22 @@
 <?php
-parse_str($_SERVER['QUERY_STRING'], $output);
+    parse_str($_SERVER['QUERY_STRING'], $output);
-$basepath = '/opt/files';
+    if (isset($output['filename'])) {
-$filename = $output['filename'];
+        $filename = $output['filename'];
-$path=$basepath . '/'. $filename;
+        $basepath = '/opt/files';
-$realpath = realpath($path);
+        $path = $basepath . '/'. $filename;
-if ($realpath === false) {
+        $realpath = realpath($path);
-    echo "File does not exist.";
+        if ($realpath === false) {
-}
+            http_response_code(404);
-elseif (strpos(realpath($path), $basepath) !== 0) {
+            exit("File does not exist");
-    echo "Wrong folder path.";
+        }
-} else {
+        elseif (strpos(realpath($path), $basepath) !== 0) {
+            http_response_code(422);
+            exit("Wrong folder path");
+        } else {
             echo file_get_contents($path);
-}
+        }
+    } else {
+        http_response_code(400);
+        exit("Wrong query key specified, must be 'filename=FILENAME");
+    }
 ?>

deployment/helm-k8s/resources/index.php

@@ -0,0 +1,72 @@
+<?php
+
+    $basepath = '/opt/files';
+    $objects = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($basepath, RecursiveIteratorIterator::SELF_FIRST));
+    iterateDirectory($objects, $basepath);
+
+    function iterateDirectory($objects, $basepath) 
+    {
+        $dom = new DomDocument("1.0");
+        $h3 = $dom->createElement("h3", "Index of /opt/files");
+        $dom->appendChild($h3);
+        $list = $dom->createElement("ul");
+        $dom->appendChild($list);
+        $node = $list;
+        $depth = 0;
+        foreach($objects as $name => $object){
+            $file = $object->getFilename();
+            if ($file === '.') continue;
+            if ($file === '..') continue;
+
+            if ($objects->getDepth() == $depth){
+               //just add another li as the depth hasn't changed
+               if ($object-> isDir()) {
+                  $li = $dom->createElement('li', $file);
+               } else {
+                  $li = create_href_li($file, $basepath);
+               }
+               $node->appendChild($li);
+            }
+            elseif ($objects->getDepth() > $depth){
+                //the depth increased, the last li is a non-empty folder
+                $li = $node->lastChild;
+                $ul = $dom->createElement('ul');
+                $li->appendChild($ul);
+
+                if ($object-> isDir()) {
+                   $ul->appendChild($dom->createElement('li', $object->getFilename()));
+                } else {
+                   $li = create_href_li($file, $basepath);
+                   $ul->appendChild($li);
+                }
+                $node = $ul;
+            }
+            else { //depth decreased, going back/up
+               $difference = $depth - $objects->getDepth();
+               for ($i = 0; $i < $difference; $difference--) {
+                  $node = $node->parentNode->parentNode;
+               }
+               $file = $object->getFilename();
+               $li = $dom->createElement('li', $file);
+               $node->appendChild($li);
+            }
+            $depth = $objects->getDepth();
+        }
+        echo $dom->saveHtml();
+    }
+
+    function create_href_li($file, $basepath)
+    {
+        $script = "download.php?filename=";
+        $li = $dom->createElement('li', "");
+        $a = $dom->createElement('a', $file);
+        $path = $object->getPath();
+        if (str_starts_with($path, $basepath)) {
+           $path = substr($path, strlen($basepath), strlen($path));
+        }
+        $link = $script . $path . "/" . $file;
+        $a->setAttribute('href', $link);
+        $li->appendChild($a);
+        return $li;
+    }
+?>

deployment/helm-k8s/resources/php.ini-local

@@ -0,0 +1,2 @@
+upload_max_filesize = 30M
+post_max_size = 40M ; This value must be larger than upload_max_filesize

deployment/helm-k8s/resources/upload.php

@@ -1,16 +1,70 @@
 <?PHP
-if(!empty($_FILES['uploaded_file'])) {
+  openlog("uploadFileScript", LOG_PID | LOG_PERROR, LOG_LOCAL0);
+  $MAX_SIZE = "30MB";
+  if(!empty($_FILES['uploaded_file'])) {
+    $error_code = $_FILES['uploaded_file']['name'];
+    $filename = basename($_FILES['uploaded_file']['name']);
+    if (!check_filename($filename)) {
+      logs(LOG_WARNING, "Warning: Filename is not valid, accepting names like: myfile.json, my_file-3.json");
+      closelog();
+      http_response_code(422);
+      exit("Warning: Filename is not valid.");
+    }
+    if ($error_code == 1) { //the uploaded file exceeds the upload_max_filesize directive in php.ini-local
+      logs(LOG_WARNING, "File: $filename exceeded size limit. Must be less than $MAX_SIZE");
+      closelog();
+      http_response_code(413);
+      exit("File: $filename exceeded size limit. Must be less than $MAX_SIZE");
+    }
+
+    $base_path = "/opt/files";
+    $user_full_path = "$base_path/";
+    if (isset($_POST['directory_path'])) {
+      $user_dir  = $_POST['directory_path'];
+      //the allowed characters, i.e. we do not accept e.g.: ../ . %2e%2e%2f etc.
+      if (!preg_match("/^(\/[a-zA-Z0-9]{1,}){1,}$/", $user_dir)) {
+        logs(LOG_WARNING, "Warning: Not a valid directory path");
+        closelog();
+        http_response_code(422);
+        exit("Warning: Not a valid directory path");
+      }
+
+      $user_full_path = $base_path . $user_dir;
+      if (!file_exists($user_full_path)) {
+        if (mkdir($user_full_path, 0777, true)) {
+          logs(LOG_INFO, "Directory $user_full_path created.");
+        } else {
+          logs(LOG_INFO, "Directory $user_full_path could not be created.");
+        }
+      } else {
+        logs(LOG_INFO, "Directory $user_full_path exists.");
+      } 
+    }
+
+    $real_path = realpath($user_full_path);
+    $f_size = $_FILES['uploaded_file']['size'];
+    if (move_uploaded_file($_FILES['uploaded_file']['tmp_name'], "$real_path/$filename")) {
+      $msg = "The file ". basename($filename). " (filesize: $f_size bytes) has been uploaded to $real_path";
+      logs(LOG_INFO, $msg);
+    } else {
+      $msg = "Error uploading the file: ". basename($filename). " (filesize: $f_size bytes) to $real_path";
+      logs(LOG_INFO, $msg);
+      http_response_code(500);
+    }
+  } else {
+     http_response_code(400);
+     logs(LOG_WARNING, "User specified wrong key, must be 'uploaded_file=FILENAME'");
+  }
+  closelog();
+
+  function logs($level, $msg)
   {
-    $path = "/opt/files";
+    $timestamp = date("d/M/Y H:i:s");
-    $filename = basename( $_FILES['uploaded_file']['name']);
+    syslog($level, "[$timestamp] $msg");
-    if(move_uploaded_file($_FILES['uploaded_file']['tmp_name'], "$path/$filename")) {
-      echo "The file ".  basename( $_FILES['uploaded_file']['name']). 
-      " has been uploaded";
-    } else{
-        echo "There was an error uploading the file, please try again!";
   }
+  function check_filename($name)
+  {
+    //the allowed filename string, accepts e.g. myfile.json, my-file-3.json
+    return preg_match("/^([a-zA-Z0-9-_]{1,})(.json)$/", $name);
   }
-} else {
-    echo "Please specify a file with correct key; 'uploaded_file=FILENAME'";
-}
 ?>

deployment/helm-k8s/templates/enrichment_store/configmap.yaml

@@ -1,6 +1,15 @@
 {{- if (has "enrichment_store" .Values.enabled_apps) -}}
 apiVersion: v1
 kind: ConfigMap
+metadata:
+  name: apache-conf
+  namespace: {{ .Values.namespace }}
+data:
+  php.ini-local: |-
+{{ .Files.Get "resources/php.ini-local" | indent 4 }}
+---
+apiVersion: v1
+kind: ConfigMap
 metadata:
   name: php-files
   namespace: {{ .Values.namespace }}
@@ -9,4 +18,6 @@ data:
 {{ .Files.Get "resources/upload.php" | indent 4 }}
   download.php: |-
 {{ .Files.Get "resources/download.php" | indent 4 }}
+  index.php: |-
+{{ .Files.Get "resources/index.php" | indent 4 }}
 {{- end -}}

deployment/helm-k8s/templates/enrichment_store/deployment.yaml

@@ -26,6 +26,11 @@ spec:
         - image: {{ .Values.enrichment_store.image.repository }}:{{ .Values.enrichment_store.image.tag }}
           imagePullPolicy: {{ .Values.enrichment_store.image.pullPolicy }}
           name: {{ include "siembol.enrichment_store.fullname" $ }}
+          env:
+          - name: APACHE_RUN_USER
+            value: "#82"
+          - name: APACHE_RUN_GROUP
+            value: "#82"
           ports:
             - name: file-server
               containerPort: {{ .Values.enrichment_store.containerPort }}
@@ -41,6 +46,9 @@ spec:
               mountPath: /opt/files
             - name: code
               mountPath: /var/www/html
+            - name: config
+              mountPath: "/usr/local/etc/php/conf.d/upload.ini"
+              subPath: "php.ini-local"
       restartPolicy: Always
       securityContext:
         runAsUser: {{ .Values.enrichment_store.security.user }}
@@ -52,4 +60,7 @@ spec:
         - name: files
           persistentVolumeClaim:
             claimName: {{ .Values.enrichment_store.pvc.name }}
+        - name: config
+          configMap:
+            name: apache-conf
   {{- end -}}

deployment/helm-k8s/templates/rest/deployment.yaml

@@ -31,11 +31,11 @@ spec:
           - containerPort: {{ .Values.rest.containerPort }}
         resources:
           requests:
+            memory: "256Mi"
+            cpu: "250m"
+          limits:
             memory: "512Mi"
             cpu: "500m"
-          limits:
-            memory: "1048Mi"
-            cpu: "1000m"
         securityContext:
           runAsUser: 101
         readinessProbe:

deployment/helm-k8s/templates/topology-manager/deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   labels:
     app: {{ include "siembol.manager.appname.fullname" $ }}
   name: {{ include "siembol.manager.appname.fullname" $ }}
+  namespace: {{ .Values.namespace }}
 spec:
   replicas: 1 
   selector:
@@ -32,10 +33,10 @@ spec:
         resources:
           requests:
             memory: "128Mi"
-            cpu: "500m"
+            cpu: "250m"
           limits:
             memory: "512Mi"
-            cpu: "1000m"
+            cpu: "500m"
         securityContext:
           runAsUser: 101
         readinessProbe:

deployment/helm-k8s/templates/topology-manager/serviceaccount-manager.yaml

@@ -4,11 +4,13 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: {{ .Values.manager.serviceAccountName }}
+  namespace: {{ .Values.namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: {{ include "siembol.manager.appname.fullname" $ }}-role
+  namespace: {{ .Values.namespace }}
 rules:
 - apiGroups: ["batch"]
   resources: ["jobs"]
@@ -18,6 +20,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: role-grantor-binding
+  namespace: {{ .Values.namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role

deployment/helm-k8s/values.yaml

@@ -89,7 +89,7 @@ enrichment_store:
   appName: "enrichment-store"
   image:
     repository: php
-    tag: 7.2-apache
+    tag: 8.0-apache
     pullPolicy: Always
   containerPort: 80
   service:

deployment/quickstart_install/ps-scripts/demoInstall.ps1

@@ -33,7 +33,7 @@ function Git-Details {
 function Init-Zookeeper-Nodes {
     $zookeeperNodes = "/siembol/synchronise", "/siembol/alerts", "/siembol/correlation_alerts", "/siembol/parser_configs", "/siembol/cache", "/siembol/enrichment_rules", "/siembol/enrichment_tables"
     Write-Output "Creating Zookeeper nodes "
-    $POD_NAME=$(kubectl get pods --namespace $NAMESPACE -l "app.kubernetes.io/name=zookeeper,app.kubernetes.io/instance=siembol-zookeeper,app.kubernetes.io/component=zookeeper" -o jsonpath="{.items[0].metadata.name}")
+    $POD_NAME=$(kubectl get pods --namespace $NAMESPACE -l "app.kubernetes.io/component=zookeeper,app.kubernetes.io/instance=storm,app.kubernetes.io/name=zookeeper" -o jsonpath="{.items[0].metadata.name}")
     kubectl exec -it $POD_NAME -n $NAMESPACE -- zkCli.sh create /siembol 1> $null
     Foreach($node in $zookeeperNodes) {
         kubectl exec -it $POD_NAME -n $NAMESPACE -- zkCli.sh create $node 1> $null
@@ -46,7 +46,7 @@ function Init-Zookeeper-Nodes {
 Write-Output "************** Install Script For Demo **************"
 Write-Output "*****************************************************"
 
-$zookeeper_status=$(kubectl get pods --namespace $NAMESPACE -l "app.kubernetes.io/name=zookeeper,app.kubernetes.io/instance=siembol-zookeeper,app.kubernetes.io/component=zookeeper" -o jsonpath="{.items[0].status.containerStatuses[0].ready}")
+$zookeeper_status=$(kubectl get pods --namespace $NAMESPACE -l "app.kubernetes.io/component=zookeeper,app.kubernetes.io/instance=storm,app.kubernetes.io/name=zookeeper" -o jsonpath="{.items[0].status.containerStatuses[0].ready}")
 if ($zookeeper_status -eq 'True')  {
     Git-Details
     Write-Output "************************************************************"

deployment/quickstart_install/ps-scripts/dependencies.ps1

@@ -7,8 +7,6 @@ helm repo add gresearch https://g-research.github.io/charts
 helm repo update
 
 
-helm install siembol-zookeeper bitnami/zookeeper --namespace $namespace
-
 helm install kafka bitnami/kafka --namespace $namespace
 
 helm install storm gresearch/storm --namespace $namespace `
@@ -20,7 +18,8 @@ helm install storm gresearch/storm --namespace $namespace `
     --set supervisor.childopts="-Xmx1g" `
     --set nimbus.image.tag=2.3.0 `
     --set supervisor.slots=3 `
-    --set ui.image.tag=2.3.0
+    --set ui.image.tag=2.3.0 `
+    --set zookeeper.fullnameOverride="siembol-zookeeper"
 
 
 Write-Output "************************************************************"

deployment/quickstart_install/sh-scripts/demoInstall.sh

@@ -35,7 +35,7 @@ git_details () {
 init_zookeeper_nodes () {
     declare -a ZookeeperNodes=("/siembol/synchronise" "/siembol/alerts" "/siembol/correlation_alerts" "/siembol/parser_configs" "/siembol/cache") 
     echo "Creating Zookeeper nodes "
-    POD_NAME=$(kubectl get pods --namespace $NAMESPACE -l "app.kubernetes.io/name=zookeeper,app.kubernetes.io/instance=siembol-zookeeper,app.kubernetes.io/component=zookeeper" -o jsonpath="{.items[0].metadata.name}")
+    POD_NAME=$(kubectl get pods --namespace $NAMESPACE -l "app.kubernetes.io/component=zookeeper,app.kubernetes.io/instance=storm,app.kubernetes.io/name=zookeeper" -o jsonpath="{.items[0].metadata.name}")
     kubectl exec -it $POD_NAME -n $NAMESPACE -- zkCli.sh create /siembol 1> /dev/null
     for node in "${ZookeeperNodes[@]}"; do
         kubectl exec -it $POD_NAME -n $NAMESPACE -- zkCli.sh create $node 1> /dev/null
@@ -48,7 +48,7 @@ init_zookeeper_nodes () {
 echo "************** Install Script For Demo **************"
 echo "*****************************************************"
 
-zookeeper_status=$(kubectl get pods --namespace $NAMESPACE -l "app.kubernetes.io/name=zookeeper,app.kubernetes.io/instance=siembol-zookeeper,app.kubernetes.io/component=zookeeper" -o jsonpath="{.items[0].status.containerStatuses[0].ready}")
+zookeeper_status=$(kubectl get pods --namespace $NAMESPACE -l "app.kubernetes.io/component=zookeeper,app.kubernetes.io/instance=storm,app.kubernetes.io/name=zookeeper" -o jsonpath="{.items[0].status.containerStatuses[0].ready}")
 if [ "$zookeeper_status" = true ]; then 
     git_details
     echo "************************************************************"
@@ -61,3 +61,4 @@ fi
 echo "************************************************************"
 echo "******  You can now deploy siembol from helm charts   ******"
 echo "************************************************************"
+

deployment/quickstart_install/sh-scripts/dependencies.sh

@@ -6,10 +6,15 @@ helm repo add bitnami https://charts.bitnami.com/bitnami
 helm repo add gresearch https://g-research.github.io/charts
 helm repo update
 
-helm install siembol-zookeeper bitnami/zookeeper -n=siembol
-
 helm install kafka bitnami/kafka -n=siembol
-helm install storm gresearch/storm -n=siembol --set supervisor.replicaCount=1
+helm install storm gresearch/storm -n=siembol \
+    --set supervisor.replicaCount=1  \
+    --set supervisor.image.tag=2.3.0 \
+    --set supervisor.childopts="-Xmx1g" \
+    --set supervisor.slots=3 \
+    --set nimbus.image.tag=2.3.0 \
+    --set ui.image.tag=2.3.0 \
+    --set zookeeper.fullnameOverride="siembol-zookeeper"
 
 
 echo "************************************************************"