Merge pull request #211 from kerberos-io/feature/minimize-sd-view-image

feature/minimize-sd-view-image

README.md

@@ -208,6 +208,8 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_REGION_POLYGON`                      | A single polygon set for motion detection: "x1,y1;x2,y2;x3,y3;...                               | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_RTSP`               | Full-HD RTSP endpoint to the camera you're targetting.                                          | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_SUB_RTSP`           | Sub-stream RTSP endpoint used for livestreaming (WebRTC).                                       | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_BASE_WIDTH`         | Force a specific width resolution for live view processing.                                     | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_BASE_HEIGHT`        | Force a specific height resolution for live view processing.                                    | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_ONVIF`              | Mark as a compliant ONVIF device.                                                               | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_ONVIF_XADDR`        | ONVIF endpoint/address running on the camera.                                                   | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_ONVIF_USERNAME`     | ONVIF username to authenticate against.                                                         | ""                             |

machinery/data/config/config.json

@@ -14,7 +14,9 @@
 		"ipcamera": {
 			"rtsp": "",
 			"sub_rtsp": "",
-			"fps": ""
+			"fps": "",
+			"base_width": 640,
+			"base_height": 0
 		},
 		"usbcamera": {
 			"device": ""
@@ -26,7 +28,7 @@
 		"recording": "true",
 		"snapshots": "true",
 		"liveview": "true",
-		"liveview_chunking": "true",
+		"liveview_chunking": "false",
 		"motion": "true",
 		"postrecording": 20,
 		"prerecording": 10,
@@ -120,4 +122,4 @@
 	"signing": {},
 	"realtimeprocessing": "false",
 	"realtimeprocessing_topic": ""
-}
+}

machinery/go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/kerberos-io/joy4 v1.0.64
 	github.com/kerberos-io/onvif v1.0.0
 	github.com/minio/minio-go/v6 v6.0.57
+	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/pion/interceptor v0.1.40
 	github.com/pion/rtp v1.8.19
@@ -41,6 +42,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0
 	go.opentelemetry.io/otel/sdk v1.36.0
+	go.opentelemetry.io/otel/trace v1.36.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
@@ -118,7 +120,6 @@ require (
 	github.com/ziutek/mymysql v1.5.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/otel/metric v1.36.0 // indirect
-	go.opentelemetry.io/otel/trace v1.36.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.6.0 // indirect
 	golang.org/x/arch v0.16.0 // indirect
 	golang.org/x/crypto v0.38.0 // indirect

machinery/go.sum

@@ -847,6 +847,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
 github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=

machinery/src/capture/gortsplib.go

@@ -654,7 +654,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / g.Streams[g.VideoH265Index].FPS
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH264Index].GopSize = gopSize
-					log.Log.Info(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
+					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
 						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
 					preRecording := configuration.Config.Capture.PreRecording
 					if preRecording > 0 && int(gopDuration) > 0 {
@@ -799,7 +799,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / g.Streams[g.VideoH265Index].FPS
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH265Index].GopSize = gopSize
-					log.Log.Info(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
+					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
 						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
 					preRecording := configuration.Config.Capture.PreRecording
 					if preRecording > 0 && int(gopDuration) > 0 {

machinery/src/capture/main.go

@@ -727,7 +727,7 @@ func VerifyCamera(c *gin.Context) {
 	}
 }
 
-func Base64Image(captureDevice *Capture, communication *models.Communication) string {
+func Base64Image(captureDevice *Capture, communication *models.Communication, configuration *models.Configuration) string {
 	// We'll try to get a snapshot from the camera.
 	var queue *packets.Queue
 	var cursor *packets.QueueCursor
@@ -757,7 +757,8 @@ func Base64Image(captureDevice *Capture, communication *models.Communication) st
 				var img image.YCbCr
 				img, err = (*rtspClient).DecodePacket(pkt)
 				if err == nil {
-					bytes, _ := utils.ImageToBytes(&img)
+					imageResized, _ := utils.ResizeImage(&img, uint(configuration.Config.Capture.IPCamera.BaseWidth), uint(configuration.Config.Capture.IPCamera.BaseHeight))
+					bytes, _ := utils.ImageToBytes(imageResized)
 					encodedImage = base64.StdEncoding.EncodeToString(bytes)
 					break
 				} else {

machinery/src/cloud/Cloud.go

@@ -706,29 +706,12 @@ func HandleLiveStreamSD(livestreamCursor *packets.QueueCursor, configuration *mo
 				log.Log.Info("cloud.HandleLiveStreamSD(): Sending base64 encoded images to MQTT.")
 				img, err := rtspClient.DecodePacket(pkt)
 				if err == nil {
-					bytes, _ := utils.ImageToBytes(&img)
+					imageResized, _ := utils.ResizeImage(&img, uint(config.Capture.IPCamera.BaseWidth), uint(config.Capture.IPCamera.BaseHeight))
+					bytes, _ := utils.ImageToBytes(imageResized)
 
 					chunking := config.Capture.LiveviewChunking
-					if chunking == "false" {
-						encoded := base64.StdEncoding.EncodeToString(bytes)
 
-						valueMap := make(map[string]interface{})
-						valueMap["image"] = encoded
-						message := models.Message{
-							Payload: models.Payload{
-								Action:   "receive-sd-stream",
-								DeviceId: configuration.Config.Key,
-								Value:    valueMap,
-							},
-						}
-						payload, err := models.PackageMQTTMessage(configuration, message)
-						if err == nil {
-							mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
-						} else {
-							log.Log.Info("cloud.HandleLiveStreamSD(): something went wrong while sending acknowledge config to hub: " + string(payload))
-						}
-
-					} else {
+					if chunking == "true" {
 
 						// Split encoded image into chunks of 2kb
 						// This is to prevent the MQTT message to be too large.
@@ -737,7 +720,7 @@ func HandleLiveStreamSD(livestreamCursor *packets.QueueCursor, configuration *mo
 						// To avoid base64 encoding, just send the raw []byte chunks as you do here.
 						// If you want to avoid base64, make sure the receiver can handle binary payloads.
 
-						chunkSize := 2 * 1024 // 2KB chunks
+						chunkSize := 25 * 1024 // 25KB chunks
 						var chunks [][]byte
 						for i := 0; i < len(bytes); i += chunkSize {
 							end := i + chunkSize
@@ -770,12 +753,32 @@ func HandleLiveStreamSD(livestreamCursor *packets.QueueCursor, configuration *mo
 							if err == nil {
 								mqttClient.Publish("kerberos/hub/"+hubKey+"/"+deviceId, 1, false, payload)
 								log.Log.Infof("cloud.HandleLiveStreamSD(): sent chunk %d/%d to MQTT topic kerberos/hub/%s/%s", i+1, len(chunks), hubKey, deviceId)
+								time.Sleep(33 * time.Millisecond) // Sleep to avoid flooding the MQTT broker with messages
 							} else {
 								log.Log.Info("cloud.HandleLiveStreamSD(): something went wrong while sending acknowledge config to hub: " + string(payload))
 							}
 						}
+					} else {
+
+						valueMap := make(map[string]interface{})
+						valueMap["image"] = bytes
+						message := models.Message{
+							Payload: models.Payload{
+								Action:   "receive-sd-stream",
+								DeviceId: configuration.Config.Key,
+								Value:    valueMap,
+							},
+						}
+						payload, err := models.PackageMQTTMessage(configuration, message)
+						if err == nil {
+							mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
+						} else {
+							log.Log.Info("cloud.HandleLiveStreamSD(): something went wrong while sending acknowledge config to hub: " + string(payload))
+						}
+
 					}
 				}
+				time.Sleep(1000 * time.Millisecond) // Sleep to avoid flooding the MQTT broker with messages
 			}
 
 		} else {
@@ -862,7 +865,8 @@ func HandleRealtimeProcessing(processingCursor *packets.QueueCursor, configurati
 				log.Log.Info("cloud.RealtimeProcessing(): Sending base64 encoded images to MQTT.")
 				img, err := rtspClient.DecodePacket(pkt)
 				if err == nil {
-					bytes, _ := utils.ImageToBytes(&img)
+					imageResized, _ := utils.ResizeImage(&img, uint(config.Capture.IPCamera.BaseWidth), uint(config.Capture.IPCamera.BaseHeight))
+					bytes, _ := utils.ImageToBytes(imageResized)
 					encoded := base64.StdEncoding.EncodeToString(bytes)
 
 					valueMap := make(map[string]interface{})

machinery/src/components/Kerberos.go

@@ -173,6 +173,18 @@ func RunAgent(configDirectory string, configuration *models.Configuration, commu
 	configuration.Config.Capture.IPCamera.Width = width
 	configuration.Config.Capture.IPCamera.Height = height
 
+	// Set the liveview width and height, this is used for the liveview and motion regions (drawing on the hub).
+	baseWidth := config.Capture.IPCamera.BaseWidth
+	baseHeight := config.Capture.IPCamera.BaseHeight
+	// If the liveview height is not set, we will calculate it based on the width and aspect ratio of the camera.
+	if baseWidth > 0 && baseHeight == 0 {
+		widthAspectRatio := float64(baseWidth) / float64(width)
+		configuration.Config.Capture.IPCamera.BaseHeight = int(float64(height) * widthAspectRatio)
+	} else {
+		configuration.Config.Capture.IPCamera.BaseHeight = height
+		configuration.Config.Capture.IPCamera.BaseWidth = width
+	}
+
 	// Set the SPS and PPS values in the configuration.
 	configuration.Config.Capture.IPCamera.SPSNALUs = [][]byte{videoStream.SPS}
 	configuration.Config.Capture.IPCamera.PPSNALUs = [][]byte{videoStream.PPS}
@@ -226,6 +238,19 @@ func RunAgent(configDirectory string, configuration *models.Configuration, commu
 		// Set config values as well
 		configuration.Config.Capture.IPCamera.SubWidth = width
 		configuration.Config.Capture.IPCamera.SubHeight = height
+
+		// If we have a substream, we need to set the width and height of the substream. (so we will override above information)
+		// Set the liveview width and height, this is used for the liveview and motion regions (drawing on the hub).
+		baseWidth := config.Capture.IPCamera.BaseWidth
+		baseHeight := config.Capture.IPCamera.BaseHeight
+		// If the liveview height is not set, we will calculate it based on the width and aspect ratio of the camera.
+		if baseWidth > 0 && baseHeight == 0 {
+			widthAspectRatio := float64(baseWidth) / float64(width)
+			configuration.Config.Capture.IPCamera.BaseHeight = int(float64(height) * widthAspectRatio)
+		} else {
+			configuration.Config.Capture.IPCamera.BaseHeight = height
+			configuration.Config.Capture.IPCamera.BaseWidth = width
+		}
 	}
 
 	// We are creating a queue to store the RTSP frames in, these frames will be
@@ -676,7 +701,7 @@ func MakeRecording(c *gin.Context, communication *models.Communication) {
 // @Success 200
 func GetSnapshotBase64(c *gin.Context, captureDevice *capture.Capture, configuration *models.Configuration, communication *models.Communication) {
 	// We'll try to get a snapshot from the camera.
-	base64Image := capture.Base64Image(captureDevice, communication)
+	base64Image := capture.Base64Image(captureDevice, communication, configuration)
 	if base64Image != "" {
 		communication.Image = base64Image
 	}
@@ -698,7 +723,8 @@ func GetSnapshotRaw(c *gin.Context, captureDevice *capture.Capture, configuratio
 	image := capture.JpegImage(captureDevice, communication)
 
 	// encode image to jpeg
-	bytes, _ := utils.ImageToBytes(&image)
+	imageResized, _ := utils.ResizeImage(&image, uint(configuration.Config.Capture.IPCamera.BaseWidth), uint(configuration.Config.Capture.IPCamera.BaseHeight))
+	bytes, _ := utils.ImageToBytes(imageResized)
 
 	// Return image/jpeg
 	c.Data(200, "image/jpeg", bytes)
@@ -713,7 +739,7 @@ func GetSnapshotRaw(c *gin.Context, captureDevice *capture.Capture, configuratio
 // @Success 200
 func GetConfig(c *gin.Context, captureDevice *capture.Capture, configuration *models.Configuration, communication *models.Communication) {
 	// We'll try to get a snapshot from the camera.
-	base64Image := capture.Base64Image(captureDevice, communication)
+	base64Image := capture.Base64Image(captureDevice, communication, configuration)
 	if base64Image != "" {
 		communication.Image = base64Image
 	}

machinery/src/computervision/main.go

@@ -63,16 +63,34 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 			}
 		}
 
+		// A user might have set the base width and height for the IPCamera.
+		// This means also the polygon coordinates are set to a specific width and height (which might be different than the actual packets
+		// received from the IPCamera). So we will resize the polygon coordinates to the base width and height.
+		baseWidthRatio := 1.0
+		baseHeightRatio := 1.0
+		baseWidth := config.Capture.IPCamera.BaseWidth
+		baseHeight := config.Capture.IPCamera.BaseHeight
+		if baseWidth > 0 && baseHeight > 0 {
+			// We'll get the first image to calculate the ratio
+			img := imageArray[0]
+			if img != nil {
+				bounds := img.Bounds()
+				rows := bounds.Dy()
+				cols := bounds.Dx()
+				baseWidthRatio = float64(cols) / float64(baseWidth)
+				baseHeightRatio = float64(rows) / float64(baseHeight)
+			}
+		}
+
 		// Calculate mask
 		var polyObjects []geo.Polygon
-
 		if config.Region != nil {
 			for _, polygon := range config.Region.Polygon {
 				coords := polygon.Coordinates
 				poly := geo.Polygon{}
 				for _, c := range coords {
-					x := c.X
-					y := c.Y
+					x := c.X * baseWidthRatio
+					y := c.Y * baseHeightRatio
 					p := geo.NewPoint(x, y)
 					if !poly.Contains(p) {
 						poly.Add(p)

machinery/src/config/main.go

@@ -239,7 +239,15 @@ func OverrideWithEnvironmentVariables(configuration *models.Configuration) {
 				configuration.Config.Capture.IPCamera.SubRTSP = value
 				break
 
-				/* ONVIF connnection settings */
+			/* Base width and height for the liveview and motion regions */
+			case "AGENT_CAPTURE_IPCAMERA_BASE_WIDTH":
+				configuration.Config.Capture.IPCamera.BaseWidth, _ = strconv.Atoi(value)
+				break
+			case "AGENT_CAPTURE_IPCAMERA_BASE_HEIGHT":
+				configuration.Config.Capture.IPCamera.BaseHeight, _ = strconv.Atoi(value)
+				break
+
+			/* ONVIF connnection settings */
 			case "AGENT_CAPTURE_IPCAMERA_ONVIF":
 				configuration.Config.Capture.IPCamera.ONVIF = value
 				break

machinery/src/models/Config.go

@@ -79,13 +79,18 @@ type Capture struct {
 // IPCamera configuration, such as the RTSP url of the IPCamera and the FPS.
 // Also includes ONVIF integration
 type IPCamera struct {
-	RTSP          string   `json:"rtsp"`
-	Width         int      `json:"width"`
-	Height        int      `json:"height"`
-	FPS           string   `json:"fps"`
-	SubRTSP       string   `json:"sub_rtsp"`
-	SubWidth      int      `json:"sub_width"`
-	SubHeight     int      `json:"sub_height"`
+	RTSP   string `json:"rtsp"`
+	Width  int    `json:"width"`
+	Height int    `json:"height"`
+	FPS    string `json:"fps"`
+
+	SubRTSP   string `json:"sub_rtsp"`
+	SubWidth  int    `json:"sub_width"`
+	SubHeight int    `json:"sub_height"`
+
+	BaseWidth  int `json:"base_width"`
+	BaseHeight int `json:"base_height"`
+
 	SubFPS        string   `json:"sub_fps"`
 	ONVIF         string   `json:"onvif,omitempty" bson:"onvif"`
 	ONVIFXAddr    string   `json:"onvif_xaddr" bson:"onvif_xaddr"`

machinery/src/routers/http/routes.go

@@ -15,7 +15,7 @@ import (
 func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configDirectory string, configuration *models.Configuration, communication *models.Communication, captureDevice *capture.Capture) *gin.RouterGroup {
 
 	r.GET("/ws", func(c *gin.Context) {
-		websocket.WebsocketHandler(c, communication, captureDevice)
+		websocket.WebsocketHandler(c, configuration, communication, captureDevice)
 	})
 
 	// This is legacy should be removed in future! Now everything

machinery/src/routers/mqtt/main.go

@@ -389,14 +389,6 @@ func HandleRequestConfig(mqttClient mqtt.Client, hubKey string, payload models.P
 			// Copy the config, as we don't want to share the encryption part.
 			deepCopy := configuration.Config
 
-			// We need a fix for the width and height if a substream.
-			// The ROI requires the width and height of the sub stream.
-			if configuration.Config.Capture.IPCamera.SubRTSP != "" &&
-				configuration.Config.Capture.IPCamera.SubRTSP != configuration.Config.Capture.IPCamera.RTSP {
-				deepCopy.Capture.IPCamera.Width = configuration.Config.Capture.IPCamera.SubWidth
-				deepCopy.Capture.IPCamera.Height = configuration.Config.Capture.IPCamera.SubHeight
-			}
-
 			var configMap map[string]interface{}
 			inrec, _ := json.Marshal(deepCopy)
 			json.Unmarshal(inrec, &configMap)

machinery/src/routers/websocket/main.go

@@ -49,7 +49,7 @@ var upgrader = websocket.Upgrader{
 	},
 }
 
-func WebsocketHandler(c *gin.Context, communication *models.Communication, captureDevice *capture.Capture) {
+func WebsocketHandler(c *gin.Context, configuration *models.Configuration, communication *models.Communication, captureDevice *capture.Capture) {
 	w := c.Writer
 	r := c.Request
 	conn, err := upgrader.Upgrade(w, r, nil)
@@ -112,7 +112,7 @@ func WebsocketHandler(c *gin.Context, communication *models.Communication, captu
 
 						ctx, cancel := context.WithCancel(context.Background())
 						sockets[clientID].Cancels["stream-sd"] = cancel
-						go ForwardSDStream(ctx, clientID, sockets[clientID], communication, captureDevice)
+						go ForwardSDStream(ctx, clientID, sockets[clientID], configuration, communication, captureDevice)
 					}
 				}
 			}
@@ -131,7 +131,7 @@ func WebsocketHandler(c *gin.Context, communication *models.Communication, captu
 	}
 }
 
-func ForwardSDStream(ctx context.Context, clientID string, connection *Connection, communication *models.Communication, captureDevice *capture.Capture) {
+func ForwardSDStream(ctx context.Context, clientID string, connection *Connection, configuration *models.Configuration, communication *models.Communication, captureDevice *capture.Capture) {
 
 	var queue *packets.Queue
 	var cursor *packets.QueueCursor
@@ -159,7 +159,10 @@ logreader:
 				var img image.YCbCr
 				img, err = (*rtspClient).DecodePacket(pkt)
 				if err == nil {
-					bytes, _ := utils.ImageToBytes(&img)
+					config := configuration.Config
+					// Resize the image to the base width and height
+					imageResized, _ := utils.ResizeImage(&img, uint(config.Capture.IPCamera.BaseWidth), uint(config.Capture.IPCamera.BaseHeight))
+					bytes, _ := utils.ImageToBytes(imageResized)
 					encodedImage = base64.StdEncoding.EncodeToString(bytes)
 				} else {
 					continue

machinery/src/utils/main.go

@@ -21,6 +21,8 @@ import (
 	"github.com/kerberos-io/agent/machinery/src/encryption"
 	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+
+	"github.com/nfnt/resize"
 )
 
 const VERSION = "3.5.0"
@@ -401,9 +403,31 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 	}
 }
 
-func ImageToBytes(img image.Image) ([]byte, error) {
+func ImageToBytes(img *image.Image) ([]byte, error) {
 	buffer := new(bytes.Buffer)
 	w := bufio.NewWriter(buffer)
-	err := jpeg.Encode(w, img, &jpeg.Options{Quality: 15})
+	err := jpeg.Encode(w, *img, &jpeg.Options{Quality: 35})
+	log.Log.Debug("ImageToBytes() - buffer size: " + strconv.Itoa(buffer.Len()))
 	return buffer.Bytes(), err
 }
+
+func ResizeImage(img image.Image, newWidth uint, newHeight uint) (*image.Image, error) {
+	if img == nil {
+		return nil, errors.New("image is nil")
+	}
+
+	// resize to width 640 using Lanczos resampling
+	// and preserve aspect ratio
+	m := resize.Resize(newWidth, newHeight, img, resize.Lanczos3)
+	return &m, nil
+}
+
+func ResizeHeightWithAspectRatio(newWidth int, width int, height int) (int, int) {
+	if newWidth <= 0 || width <= 0 || height <= 0 {
+		return width, height
+	}
+	// Calculate the new height based on the aspect ratio
+	newHeight := (newWidth * height) / width
+	// Return the new dimensions
+	return newWidth, newHeight
+}

machinery/src/video/mp4.go

@@ -64,15 +64,14 @@ func NewMP4(fileName string, spsNALUs [][]byte, ppsNALUs [][]byte, vpsNALUs [][]
 	init := mp4ff.NewMP4Init()
 
 	// Add a free box to the init segment
-	// Prepend a free box to the init segment with a size of 1000
-	freeBoxSize := 2048
+	// Prepend a free box to the init segment with a size of 4096 bytes, so we can overwrite it later with the actual init segment.
+	freeBoxSize := 4096
 	free := mp4ff.NewFreeBox(make([]byte, freeBoxSize))
 	init.AddChild(free)
 
 	// Create a writer
 	ofd, err := os.Create(fileName)
 	if err != nil {
-		panic(err)
 	}
 
 	// Create a buffered writer
@@ -82,7 +81,6 @@ func NewMP4(fileName string, spsNALUs [][]byte, ppsNALUs [][]byte, vpsNALUs [][]
 	// so we can overwrite it later with the actual init segment.
 	err = init.Encode(bufferedWriter)
 	if err != nil {
-		panic(err)
 	}
 
 	return &MP4{
@@ -142,7 +140,7 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 			mp4.MoofBoxSizes = append(mp4.MoofBoxSizes, int64(mp4.Segment.Size()))
 			err := mp4.Segment.Encode(mp4.Writer)
 			if err != nil {
-				return err
+				log.Log.Error("mp4.AddSampleToTrack(): error encoding segment: " + err.Error())
 			}
 			mp4.Segments = append(mp4.Segments, mp4.Segment)
 		}
@@ -158,6 +156,7 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 		// Create a video fragment
 		multiTrackFragment, err := mp4ff.CreateMultiTrackFragment(uint32(mp4.SegmentCount), mp4.TrackIDs) // Assuming 1 for video track and 2 for audio track
 		if err != nil {
+			log.Log.Error("mp4.AddSampleToTrack(): error creating multi track fragment: " + err.Error())
 		}
 		mp4.MultiTrackFragment = multiTrackFragment
 		seg.AddFragment(multiTrackFragment)
@@ -175,9 +174,10 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 
 			var lengthPrefixed []byte
 			var err error
-			if mp4.VideoTrackName == "H264" || mp4.VideoTrackName == "AVC1" { // Convert Annex B to length-prefixed NAL units if H264
+			switch mp4.VideoTrackName {
+			case "H264", "AVC1": // Convert Annex B to length-prefixed NAL units if H264
 				lengthPrefixed, err = annexBToLengthPrefixed(data)
-			} else if mp4.VideoTrackName == "H265" || mp4.VideoTrackName == "HVC1" { // Convert H265 Annex B to length-prefixed NAL units
+			case "H265", "HVC1": // Convert H265 Annex B to length-prefixed NAL units
 				lengthPrefixed, err = annexBToLengthPrefixed(data)
 			}
 
@@ -187,14 +187,12 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 					log.Log.Debug("Adding sample to track " + fmt.Sprintf("%d, PTS: %d, Duration: %d, size: %d, Keyframe: %t", trackID, pts, duration, len(lengthPrefixed), isKeyframe))
 
 					mp4.LastVideoSampleDTS = duration
-					//fmt.Printf("Adding sample to track %d, PTS: %d, Duration: %d, size: %d, Keyframe: %t\n", trackID, pts, duration, len(mp4.VideoFullSample.Data), isKeyframe)
 					mp4.VideoTotalDuration += duration
 					mp4.VideoFullSample.DecodeTime = mp4.VideoTotalDuration - duration
 					mp4.VideoFullSample.Sample.Dur = uint32(duration)
 					err := mp4.MultiTrackFragment.AddFullSampleToTrack(*mp4.VideoFullSample, trackID)
 					if err != nil {
-						//log.Printf("Error adding sample to track %d: %v", trackID, err)
-						return err
+						log.Log.Error("mp4.AddSampleToTrack(): error adding sample to track " + fmt.Sprintf("%d: %v", trackID, err))
 					}
 				}
 
@@ -261,33 +259,6 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 
 func (mp4 *MP4) Close(config *models.Config) {
 
-	// Add the last sample to the track, we will predict the duration based on the last sample
-	// We are not insert the last sample as we might corrupt playback (as we do not know accurately the next PTS).
-	// In theory it means we will lose the last sample, so there is millisecond dataloss, but it is better than corrupting playback.
-	// We could this by using a delayed packet reader, and look for the next PTS (closest one), but that would require a lot of memory and CPU.
-
-	/*duration := uint64(0)
-	trackID := uint32(1)
-	if mp4.SampleType == "video" {
-		duration = mp4.LastVideoSampleDTS
-		trackID = uint32(mp4.VideoTrack)
-	} else if mp4.SampleType == "audio" {
-		duration = 21 //mp4.LastAudioSampleDTS
-
-	} else {
-		log.Println("mp4.Close(): unknown sample type, cannot calculate duration")
-	}
-
-	if duration > 0 {
-		mp4.VideoTotalDuration += duration
-		mp4.VideoFullSample.DecodeTime = mp4.VideoTotalDuration - duration
-		mp4.VideoFullSample.Sample.Dur = uint32(duration)
-		err := mp4.MultiTrackFragment.AddFullSampleToTrack(*mp4.VideoFullSample, trackID)
-		if err != nil {
-		}
-		mp4.Segments = append(mp4.Segments, mp4.Segment)
-	}*/
-
 	if mp4.VideoTotalDuration == 0 && mp4.AudioTotalDuration == 0 {
 		log.Log.Error("mp4.Close(): no video or audio samples added, cannot create MP4 file")
 	}
@@ -296,7 +267,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 	if mp4.Segment != nil {
 		err := mp4.Segment.Encode(mp4.Writer)
 		if err != nil {
-			panic(err)
+			log.Log.Error("mp4.Close(): error encoding last segment: " + err.Error())
 		}
 	}
 
@@ -304,7 +275,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 	defer mp4.FileWriter.Close()
 
 	// Now we have all the moof and mdat boxes written to the file.
-	// We can now generate the ftyp and moov boxes, and replace it with the free box we added earlier (size of 10008 bytes).
+	// We can now generate the ftyp and moov boxes, and replace it with the free box we added earlier (size of 2048 bytes).
 	init := mp4ff.NewMP4Init()
 
 	// Create a new ftyp box
@@ -337,22 +308,21 @@ func (mp4 *MP4) Close(config *models.Config) {
 	init.Moov.AddChild(mvex)
 
 	// Add a track for the video
-	if mp4.VideoTrackName == "H264" || mp4.VideoTrackName == "AVC1" {
+	switch mp4.VideoTrackName {
+	case "H264", "AVC1":
 		init.AddEmptyTrack(videoTimescale, "video", "und")
 		includePS := true
 		err := init.Moov.Traks[0].SetAVCDescriptor("avc1", mp4.SPSNALUs, mp4.PPSNALUs, includePS)
 		if err != nil {
-			//panic(err)
 		}
 		init.Moov.Traks[0].Tkhd.Duration = mp4.VideoTotalDuration
 		init.Moov.Traks[0].Mdia.Hdlr.Name = "agent " + utils.VERSION
 		//init.Moov.Traks[0].Mdia.Mdhd.Duration = mp4.VideoTotalDuration
-	} else if mp4.VideoTrackName == "H265" || mp4.VideoTrackName == "HVC1" {
+	case "H265", "HVC1":
 		init.AddEmptyTrack(videoTimescale, "video", "und")
 		includePS := true
 		err := init.Moov.Traks[0].SetHEVCDescriptor("hvc1", mp4.VPSNALUs, mp4.SPSNALUs, mp4.PPSNALUs, [][]byte{}, includePS)
 		if err != nil {
-			//panic(err)
 		}
 		init.Moov.Traks[0].Tkhd.Duration = mp4.VideoTotalDuration
 		init.Moov.Traks[0].Mdia.Hdlr.Name = "agent " + utils.VERSION
@@ -372,7 +342,6 @@ func (mp4 *MP4) Close(config *models.Config) {
 		// Set the audio descriptor
 		err := init.Moov.Traks[1].SetAACDescriptor(29, audioSampleRate)
 		if err != nil {
-			//panic(err)
 		}
 		init.Moov.Traks[1].Tkhd.Duration = mp4.AudioTotalDuration
 		init.Moov.Traks[1].Mdia.Hdlr.Name = "agent " + utils.VERSION
@@ -481,16 +450,15 @@ func (mp4 *MP4) Close(config *models.Config) {
 	init.AddChild(sidx)*/
 
 	// Get a bit slice writer for the init segment
-	// Get a byte buffer of 10008 bytes to write the init segment
+	// Get a byte buffer of FreeBoxSize bytes to write the init segment
 	buffer := bytes.NewBuffer(make([]byte, 0))
 	init.Encode(buffer)
 
-	// The first 10008 bytes of the file is a free box, so we can read it and replace it with the moov box.
-	// The init box might not be 10008 bytes, so we need to read the first 10008 bytes and then replace it with the moov box.
+	// The first FreeBoxSize bytes of the file is a free box, so we can read it and replace it with the moov box.
+	// The init box might not be FreeBoxSize bytes, so we need to read the first FreeBoxSize bytes and then replace it with the moov box.
 	// while the remaining bytes are for a new free box.
 	// Write the init segment at the beginning of the file, replacing the free box
 	if _, err := mp4.FileWriter.WriteAt(buffer.Bytes(), 0); err != nil {
-		panic(err)
 	}
 
 	// Calculate the remaining size for the free box
@@ -499,10 +467,10 @@ func (mp4 *MP4) Close(config *models.Config) {
 		newFreeBox := mp4ff.NewFreeBox(make([]byte, remainingSize))
 		var freeBuf bytes.Buffer
 		if err := newFreeBox.Encode(&freeBuf); err != nil {
-			panic(err)
+			log.Log.Error("mp4.Close(): error encoding free box: " + err.Error())
 		}
 		if _, err := mp4.FileWriter.WriteAt(freeBuf.Bytes(), int64(buffer.Len())); err != nil {
-			panic(err)
+			log.Log.Error("mp4.Close(): error writing free box: " + err.Error())
 		}
 	}
 }
@@ -909,7 +877,7 @@ func SampleToAACSampleIndex(sampling int) int {
 			return i
 		}
 	}
-	panic("not Found AAC Sample Index")
+	return -1
 }
 
 func AACSampleIdxToSample(idx int) int {

ui/yarn.lock

@@ -1715,10 +1715,10 @@
     "@jridgewell/resolve-uri" "^3.0.3"
     "@jridgewell/sourcemap-codec" "^1.4.10"
 
-"@kerberos-io/ui@^1.71.0":
-  version "1.71.0"
-  resolved "https://registry.yarnpkg.com/@kerberos-io/ui/-/ui-1.71.0.tgz#06914c94e8b0982068d2099acf8158917a511bfc"
-  integrity sha512-pHCTn/iQTcQEPoCK82eJHGRn6BgzW3wgV4C+mNqdKOtLTquxL+vh7molEgC66tl3DGf7HyjSNa8LuoxYbt9TEg==
+"@kerberos-io/ui@^1.76.0":
+  version "1.77.0"
+  resolved "https://registry.yarnpkg.com/@kerberos-io/ui/-/ui-1.77.0.tgz#b748b2a9abf793ff2a9ba64ee41f84debc0ca9dc"
+  integrity sha512-CHh4jeLKwrYvJRL5PM3UEN4p2k1fqwMKgSF2U6IR4v0fE2FwPc/2Ry4zGk6pvLDFHbDpR9jUkHX+iNphvStoyQ==
   dependencies:
     "@emotion/react" "^11.10.4"
     "@emotion/styled" "^11.10.4"