Merge pull request #130 from stephb9959/main

[WIFI-11251] Now fetching device statistics in batches of 100

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "ucentral-client",
-  "version": "2.8.0(17)",
+  "version": "2.8.0(18)",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "ucentral-client",
-      "version": "2.8.0(17)",
+      "version": "2.8.0(18)",
       "license": "ISC",
       "dependencies": {
         "@chakra-ui/icons": "^2.0.11",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ucentral-client",
-  "version": "2.8.0(17)",
+  "version": "2.8.0(18)",
   "description": "",
   "private": true,
   "main": "index.tsx",

src/hooks/Network/Statistics.ts

@@ -189,29 +189,61 @@ export const useGetMacOuis = ({ macs, onError }: { macs?: string[]; onError?: (e
     onError,
   });
 
-const getStatsBetweenTimestamps = (limit: number, start?: number, end?: number, serialNumber?: string) => async () =>
+const getStatsBetweenTimestamps = async (params: {
+  start?: number;
+  end?: number;
+  serialNumber?: string;
+  offset: number;
+}) =>
   axiosGw
-    .get(`device/${serialNumber}/statistics?startDate=${start}&endDate=${end}&limit=${limit}`)
+    .get(
-    .then((response) => response.data) as Promise<{
+      `device/${params.serialNumber}/statistics?startDate=${params.start}&endDate=${params.end}&offset=${params.offset}&limit=100`,
-    data: { data: DeviceStatistics; UUID: string; recorded: number }[];
+    )
-  }>;
+    .then((response) => response.data.data as { data: DeviceStatistics; UUID: string; recorded: number }[]);
+
+const getStatsBetweenTimestampsCount = async (params: { start?: number; end?: number; serialNumber?: string }) =>
+  axiosGw
+    .get(`device/${params.serialNumber}/statistics?startDate=${params.start}&endDate=${params.end}&countOnly=true`)
+    .then((response) => response.data.count as number)
+    .catch(() => 0);
+
+const getStatsBetweenTimestampsWithProgress =
+  (params: { start?: number; end?: number; serialNumber?: string }, setProgress?: (pct: number) => void) =>
+  async () => {
+    const { start, end, serialNumber } = params;
+    if (setProgress) setProgress(0);
+    const count = await getStatsBetweenTimestampsCount(params);
+    let allStats: { data: DeviceStatistics; UUID: string; recorded: number }[] = [];
+    let offset = 0;
+    let latestResponse: { data: DeviceStatistics; UUID: string; recorded: number }[];
+    do {
+      // eslint-disable-next-line no-await-in-loop
+      latestResponse = await getStatsBetweenTimestamps({ start, end, serialNumber, offset });
+      allStats = allStats.concat(latestResponse);
+      offset += 100;
+      if (setProgress && count > 0) setProgress((allStats.length / count) * 100);
+    } while (latestResponse.length === 100);
+    if (setProgress) setProgress(100);
+
+    return allStats.sort((a, b) => a.recorded - b.recorded);
+  };
 
 export const useGetDeviceStatsWithTimestamps = ({
   serialNumber,
-  limit,
   start,
   end,
   onError,
+  setProgress,
 }: {
   serialNumber?: string;
-  limit: number;
   start?: number;
   end?: number;
   onError?: (e: AxiosError) => void;
+  setProgress?: (pct: number) => void;
 }) =>
   useQuery(
-    ['deviceStatistics', serialNumber, { limit, start, end }],
+    ['deviceStatistics', serialNumber, { start, end }],
-    getStatsBetweenTimestamps(limit, start, end, serialNumber),
+    getStatsBetweenTimestampsWithProgress({ start, end, serialNumber }, setProgress),
     {
       enabled: serialNumber !== undefined && serialNumber !== '' && start !== undefined && end !== undefined,
       staleTime: 1000 * 60,

src/pages/Device/StatisticsCard/index.tsx

@@ -5,25 +5,13 @@ import { v4 as uuid } from 'uuid';
 import StatisticsCardDatePickers from './DatePickers';
 import InterfaceChart from './InterfaceChart';
 import DeviceMemoryChart from './MemoryChart';
+import { useStatisticsCard } from './useStatisticsCard';
 import ViewLastStatsModal from './ViewLastStatsModal';
 import { RefreshButton } from 'components/Buttons/RefreshButton';
 import { Card } from 'components/Containers/Card';
 import { CardBody } from 'components/Containers/Card/CardBody';
 import { CardHeader } from 'components/Containers/Card/CardHeader';
 import { LoadingOverlay } from 'components/LoadingOverlay';
-import {
-  DeviceStatistics,
-  useGetDeviceStatsLatestHour,
-  useGetDeviceStatsWithTimestamps,
-} from 'hooks/Network/Statistics';
-
-const extractMemory = (stat: DeviceStatistics) => {
-  let used: number | undefined;
-  if (stat.unit && stat.unit.memory) {
-    used = stat.unit.memory.total - stat.unit.memory.free;
-  }
-  return { ...stat.unit?.memory, used };
-};
 
 type Props = {
   serialNumber: string;
@@ -31,20 +19,10 @@ type Props = {
 
 const DeviceStatisticsCard = ({ serialNumber }: Props) => {
   const { t } = useTranslation();
-  const [time, setTime] = React.useState<{ start: Date; end: Date } | undefined>();
+  const { time, setTime, parsedData, isLoading, selected, onSelectInterface, refresh } = useStatisticsCard({
-  const [selected, setSelected] = React.useState('memory');
-  const getStats = useGetDeviceStatsLatestHour({ serialNumber, limit: 10000 });
-  const getCustomStats = useGetDeviceStatsWithTimestamps({
     serialNumber,
-    limit: 10000,
-    start: time ? Math.floor(time.start.getTime() / 1000) : undefined,
-    end: time ? Math.floor(time.end.getTime() / 1000) : undefined,
   });
 
-  const onChange = (event: React.ChangeEvent<HTMLSelectElement>) => {
-    setSelected(event.target.value);
-  };
-
   const setNewTime = (start: Date, end: Date) => {
     setTime({ start, end });
   };
@@ -52,79 +30,6 @@ const DeviceStatisticsCard = ({ serialNumber }: Props) => {
     setTime(undefined);
   };
 
-  const parsedData = React.useMemo(() => {
-    if (!getStats.data && !getCustomStats.data) return undefined;
-
-    const data: Record<string, { tx: number[]; rx: number[]; recorded: number[]; maxRx: number; maxTx: number }> = {};
-    const memoryData = {
-      used: [] as number[],
-      buffered: [] as number[],
-      cached: [] as number[],
-      free: [] as number[],
-      total: [] as number[],
-      recorded: [] as number[],
-    };
-    const previousRx: { [key: string]: number } = {};
-    const previousTx: { [key: string]: number } = {};
-
-    const dataToLoop = getCustomStats.data ?? getStats.data;
-
-    for (const [index, stat] of dataToLoop ? dataToLoop.data.entries() : []) {
-      if (index === 0) {
-        let updated = false;
-        for (const inter of stat.data.interfaces ?? []) {
-          if (!updated && selected === 'memory') {
-            updated = true;
-            setSelected(inter.name);
-          }
-          previousRx[inter.name] = inter.counters?.rx_bytes ?? 0;
-          previousTx[inter.name] = inter.counters?.tx_bytes ?? 0;
-        }
-      } else {
-        const newMem = extractMemory(stat.data);
-        memoryData.used.push(newMem.used ?? 0);
-        memoryData.buffered.push(newMem.buffered ?? 0);
-        memoryData.cached.push(newMem.cached ?? 0);
-        memoryData.free.push(newMem.free ?? 0);
-        memoryData.total.push(newMem.total ?? 0);
-        memoryData.recorded.push(stat.recorded);
-
-        for (const inter of stat.data.interfaces ?? []) {
-          const rx = inter.counters?.rx_bytes ?? 0;
-          const tx = inter.counters?.tx_bytes ?? 0;
-          let rxDelta = rx - (previousRx[inter.name] ?? 0);
-          if (rxDelta < 0) rxDelta = 0;
-          let txDelta = tx - (previousTx[inter.name] ?? 0);
-          if (txDelta < 0) txDelta = 0;
-          if (data[inter.name] === undefined)
-            data[inter.name] = {
-              rx: [rxDelta],
-              tx: [txDelta],
-              recorded: [stat.recorded],
-              maxTx: txDelta,
-              maxRx: rxDelta,
-            };
-          else {
-            data[inter.name]?.rx.push(rxDelta);
-            data[inter.name]?.tx.push(txDelta);
-            data[inter.name]?.recorded.push(stat.recorded);
-            // @ts-ignore
-            if (data[inter.name] !== undefined && txDelta > data[inter.name].maxTx) data[inter.name].maxTx = txDelta;
-            // @ts-ignore
-            if (data[inter.name] !== undefined && rxDelta > data[inter.name].maxRx) data[inter.name].maxRx = rxDelta;
-          }
-          previousRx[inter.name] = rx;
-          previousTx[inter.name] = tx;
-        }
-      }
-    }
-
-    return {
-      interfaces: data,
-      memory: memoryData,
-    };
-  }, [getStats.data, getCustomStats.data]);
-
   const interfaces = React.useMemo(() => {
     if (!parsedData) return undefined;
 
@@ -140,13 +45,6 @@ const DeviceStatisticsCard = ({ serialNumber }: Props) => {
     return <DeviceMemoryChart data={parsedData.memory} />;
   }, [parsedData]);
 
-  const isLoading = React.useMemo(() => {
-    if (!time && getStats?.isFetching) return true;
-    if (time && getCustomStats.isFetching) return true;
-
-    return false;
-  }, [getStats, getCustomStats, time]);
-
   return (
     <Card mb={4}>
       <CardHeader display="block">
@@ -156,7 +54,7 @@ const DeviceStatisticsCard = ({ serialNumber }: Props) => {
           <HStack>
             <ViewLastStatsModal serialNumber={serialNumber} />
             <StatisticsCardDatePickers defaults={time} setTime={setNewTime} onClear={onClear} />
-            <Select value={selected} onChange={onChange}>
+            <Select value={selected} onChange={onSelectInterface}>
               {parsedData?.interfaces
                 ? Object.keys(parsedData.interfaces).map((v) => (
                     <option value={v} key={uuid()}>
@@ -168,9 +66,9 @@ const DeviceStatisticsCard = ({ serialNumber }: Props) => {
             </Select>
             <RefreshButton
               size="sm"
-              onClick={getStats.refetch}
+              onClick={refresh}
               isCompact
-              isFetching={getStats.isFetching}
+              isFetching={isLoading.isLoading}
               // @ts-ignore
               colorScheme="blue"
             />
@@ -188,12 +86,17 @@ const DeviceStatisticsCard = ({ serialNumber }: Props) => {
             </Heading>
           </Flex>
         )}
-        {!getStats?.data && isLoading ? (
+        {(!parsedData && isLoading.isLoading) || (isLoading.isLoading && isLoading.progress !== undefined) ? (
-          <Center my="auto">
+          <Center my="auto" mt="100px">
-            <Spinner size="xl" mt="100px" />
+            {isLoading.progress !== undefined && (
+              <Heading size="md" mr={2}>
+                {isLoading.progress.toFixed(2)}%
+              </Heading>
+            )}
+            <Spinner size="xl" />
           </Center>
         ) : (
-          <LoadingOverlay isLoading={isLoading}>
+          <LoadingOverlay isLoading={isLoading.isLoading}>
             <Box>
               {selected === 'memory' && memory}
               {interfaces}

src/pages/Device/StatisticsCard/useStatisticsCard.ts

@@ -0,0 +1,137 @@
+/* eslint-disable import/prefer-default-export */
+import React from 'react';
+import {
+  DeviceStatistics,
+  useGetDeviceStatsLatestHour,
+  useGetDeviceStatsWithTimestamps,
+} from 'hooks/Network/Statistics';
+
+const extractMemory = (stat: DeviceStatistics) => {
+  let used: number | undefined;
+  if (stat.unit && stat.unit.memory) {
+    used = stat.unit.memory.total - stat.unit.memory.free;
+  }
+  return { ...stat.unit?.memory, used };
+};
+
+type Props = {
+  serialNumber: string;
+};
+
+export const useStatisticsCard = ({ serialNumber }: Props) => {
+  const [selected, setSelected] = React.useState('memory');
+  const [progress, setProgress] = React.useState(0);
+  const [time, setTime] = React.useState<{ start: Date; end: Date } | undefined>();
+  const onProgressChange = React.useCallback((newProgress: number) => {
+    setProgress(newProgress);
+  }, []);
+  const getStats = useGetDeviceStatsLatestHour({ serialNumber, limit: 100 });
+  const getCustomStats = useGetDeviceStatsWithTimestamps({
+    serialNumber,
+    start: time ? Math.floor(time.start.getTime() / 1000) : undefined,
+    end: time ? Math.floor(time.end.getTime() / 1000) : undefined,
+    setProgress: onProgressChange,
+  });
+
+  const onSelectInterface = (event: React.ChangeEvent<HTMLSelectElement>) => {
+    setSelected(event.target.value);
+  };
+
+  const parsedData = React.useMemo(() => {
+    if (!getStats.data && !getCustomStats.data) return undefined;
+
+    const data: Record<string, { tx: number[]; rx: number[]; recorded: number[]; maxRx: number; maxTx: number }> = {};
+    const memoryData = {
+      used: [] as number[],
+      buffered: [] as number[],
+      cached: [] as number[],
+      free: [] as number[],
+      total: [] as number[],
+      recorded: [] as number[],
+    };
+    const previousRx: { [key: string]: number } = {};
+    const previousTx: { [key: string]: number } = {};
+
+    const dataToLoop = getCustomStats.data ?? getStats.data?.data;
+
+    for (const [index, stat] of dataToLoop ? dataToLoop.entries() : []) {
+      if (index === 0) {
+        let updated = false;
+        for (const inter of stat.data.interfaces ?? []) {
+          if (!updated && selected === 'memory') {
+            updated = true;
+            setSelected(inter.name);
+          }
+          previousRx[inter.name] = inter.counters?.rx_bytes ?? 0;
+          previousTx[inter.name] = inter.counters?.tx_bytes ?? 0;
+        }
+      } else {
+        const newMem = extractMemory(stat.data);
+        memoryData.used.push(newMem.used ?? 0);
+        memoryData.buffered.push(newMem.buffered ?? 0);
+        memoryData.cached.push(newMem.cached ?? 0);
+        memoryData.free.push(newMem.free ?? 0);
+        memoryData.total.push(newMem.total ?? 0);
+        memoryData.recorded.push(stat.recorded);
+
+        for (const inter of stat.data.interfaces ?? []) {
+          const rx = inter.counters?.rx_bytes ?? 0;
+          const tx = inter.counters?.tx_bytes ?? 0;
+          let rxDelta = rx - (previousRx[inter.name] ?? 0);
+          if (rxDelta < 0) rxDelta = 0;
+          let txDelta = tx - (previousTx[inter.name] ?? 0);
+          if (txDelta < 0) txDelta = 0;
+          if (data[inter.name] === undefined)
+            data[inter.name] = {
+              rx: [rxDelta],
+              tx: [txDelta],
+              recorded: [stat.recorded],
+              maxTx: txDelta,
+              maxRx: rxDelta,
+            };
+          else {
+            data[inter.name]?.rx.push(rxDelta);
+            data[inter.name]?.tx.push(txDelta);
+            data[inter.name]?.recorded.push(stat.recorded);
+            // @ts-ignore
+            if (data[inter.name] !== undefined && txDelta > data[inter.name].maxTx) data[inter.name].maxTx = txDelta;
+            // @ts-ignore
+            if (data[inter.name] !== undefined && rxDelta > data[inter.name].maxRx) data[inter.name].maxRx = rxDelta;
+          }
+          previousRx[inter.name] = rx;
+          previousTx[inter.name] = tx;
+        }
+      }
+    }
+
+    return {
+      interfaces: data,
+      memory: memoryData,
+    };
+  }, [getStats.data, getCustomStats.data]);
+
+  const refresh = React.useCallback(() => {
+    if (!time) getStats.refetch();
+    else getCustomStats.refetch();
+  }, [time]);
+
+  const isLoading = React.useMemo(() => {
+    if (!time && getStats?.isFetching) return { isLoading: true };
+    if (time && getCustomStats.isFetching) return { isLoading: true, progress };
+
+    return { isLoading: false };
+  }, [getStats, getCustomStats, time, progress]);
+
+  return React.useMemo(
+    () => ({
+      parsedData,
+      isLoading,
+      onSelectInterface,
+      selected,
+      time,
+      setTime,
+      refresh,
+    }),
+    [parsedData, isLoading, selected, time],
+  );
+};