initial commit

src/views/ClustersPanel.tsx

@@ -0,0 +1,112 @@
+import React, { FC, useEffect, useMemo, useState } from "react";
+import { useSigma } from "react-sigma-v2";
+import { sortBy, values, keyBy, mapValues } from "lodash";
+import { MdGroupWork } from "react-icons/md";
+import { AiOutlineCheckCircle, AiOutlineCloseCircle } from "react-icons/ai";
+
+import { Cluster, FiltersState } from "../types";
+import Panel from "./Panel";
+
+const ClustersPanel: FC<{
+  clusters: Cluster[];
+  filters: FiltersState;
+  toggleCluster: (cluster: string) => void;
+  setClusters: (clusters: Record<string, boolean>) => void;
+}> = ({ clusters, filters, toggleCluster, setClusters }) => {
+  const sigma = useSigma();
+  const graph = sigma.getGraph();
+
+  const nodesPerCluster = useMemo(() => {
+    const index: Record<string, number> = {};
+    graph.forEachNode((_, { cluster }) => (index[cluster] = (index[cluster] || 0) + 1));
+    return index;
+  }, []);
+
+  const maxNodesPerCluster = useMemo(() => Math.max(...values(nodesPerCluster)), [nodesPerCluster]);
+  const visibleClustersCount = useMemo(() => Object.keys(filters.clusters).length, [filters]);
+
+  const [visibleNodesPerCluster, setVisibleNodesPerCluster] = useState<Record<string, number>>(nodesPerCluster);
+  useEffect(() => {
+    // To ensure the graphology instance has up to data "hidden" values for
+    // nodes, we wait for next frame before reindexing. This won't matter in the
+    // UX, because of the visible nodes bar width transition.
+    requestAnimationFrame(() => {
+      const index: Record<string, number> = {};
+      graph.forEachNode((_, { cluster, hidden }) => !hidden && (index[cluster] = (index[cluster] || 0) + 1));
+      setVisibleNodesPerCluster(index);
+    });
+  }, [filters]);
+
+  const sortedClusters = useMemo(
+    () => sortBy(clusters, (cluster) => -nodesPerCluster[cluster.key]),
+    [clusters, nodesPerCluster],
+  );
+
+  return (
+    <Panel
+      title={
+        <>
+          <MdGroupWork className="text-muted" /> Clusters
+          {visibleClustersCount < clusters.length ? (
+            <span className="text-muted text-small">
+              {" "}
+              ({visibleClustersCount} / {clusters.length})
+            </span>
+          ) : (
+            ""
+          )}
+        </>
+      }
+    >
+      <p>
+        <i className="text-muted">Click a cluster to show/hide related pages from the network.</i>
+      </p>
+      <p className="buttons">
+        <button className="btn" onClick={() => setClusters(mapValues(keyBy(clusters, "key"), () => true))}>
+          <AiOutlineCheckCircle /> Check all
+        </button>{" "}
+        <button className="btn" onClick={() => setClusters({})}>
+          <AiOutlineCloseCircle /> Uncheck all
+        </button>
+      </p>
+      <ul>
+        {sortedClusters.map((cluster) => {
+          const nodesCount = nodesPerCluster[cluster.key];
+          const visibleNodesCount = visibleNodesPerCluster[cluster.key] || 0;
+          return (
+            <li
+              className="caption-row"
+              key={cluster.key}
+              title={`${nodesCount} page${nodesCount > 1 ? "s" : ""}${
+                visibleNodesCount !== nodesCount ? ` (only ${visibleNodesCount} visible)` : ""
+              }`}
+            >
+              <input
+                type="checkbox"
+                checked={filters.clusters[cluster.key] || false}
+                onChange={() => toggleCluster(cluster.key)}
+                id={`cluster-${cluster.key}`}
+              />
+              <label htmlFor={`cluster-${cluster.key}`}>
+                <span className="circle" style={{ background: cluster.color, borderColor: cluster.color }} />{" "}
+                <div className="node-label">
+                  <span>{cluster.clusterLabel}</span>
+                  <div className="bar" style={{ width: (100 * nodesCount) / maxNodesPerCluster + "%" }}>
+                    <div
+                      className="inside-bar"
+                      style={{
+                        width: (100 * visibleNodesCount) / nodesCount + "%",
+                      }}
+                    />
+                  </div>
+                </div>
+              </label>
+            </li>
+          );
+        })}
+      </ul>
+    </Panel>
+  );
+};
+
+export default ClustersPanel;

src/views/DescriptionPanel.tsx

@@ -0,0 +1,78 @@
+import React, { FC } from "react";
+import { BsInfoCircle } from "react-icons/bs";
+
+import Panel from "./Panel";
+
+const DescriptionPanel: FC = () => {
+  return (
+    <Panel
+      initiallyDeployed
+      title={
+        <>
+          <BsInfoCircle className="text-muted" /> Description
+        </>
+      }
+    >
+      <p>
+        This visualisation represents a <i>network</i> of{" "}
+        <a target="_blank" rel="noreferrer" href="https://spacy.io/usage/linguistic-features#named-entities">
+          named entities
+        </a> in English-language posts archived in a database of Telegram channels that have posted about GESARA. Each{" "}
+        <i>node</i> represents an entity, <i>edges</i> between nodes indicate that one or more posts contain both entities
+        .
+      </p>
+      <p>
+        Some social media channels were identified by researchers from{" "}
+        <a target="_blank" rel="noreferrer" href="https://www.bellingcat.com/">
+          Bellingcat
+        </a>{" "}and{" "}
+        <a target="_blank" rel="noreferrer" href="https://www.lighthousereports.nl/">
+          Lighthouse Reports
+        </a>
+        , then several rounds of snowball sampling found forwarded channels that have posted about GESARA.
+        The entities were identified using {" "}
+        <a target="_blank" rel="noreferrer" href="https://spacy.io/">
+          spaCy
+        </a>
+        .
+      </p>
+      <p>
+        This web application has been developed by{" "}
+        <a target="_blank" rel="noreferrer" href="https://www.bellingcat.com/">
+          Bellingcat
+        </a>
+        , using{" "}
+        <a target="_blank" rel="noreferrer" href="https://reactjs.org/">
+          react
+        </a>{" "}
+        and{" "}
+        <a target="_blank" rel="noreferrer" href="https://www.sigmajs.org">
+          sigma.js
+        </a>
+        . You can read the source code{" "}
+        <a target="_blank" rel="noreferrer" href="https://github.com/jacomyal/sigma.js/tree/main/demo">
+          on GitHub
+        </a>
+        .
+      </p>
+      <p>
+        Node sizes are related to the number of times the entity was mentioned in the database.
+      </p>
+      <p>
+        Nodes are colored based a{" "}
+        <a target="_blank" rel="noreferrer" href="https://arxiv.org/abs/0803.0476">
+        community detection algorithm
+        </a>.
+      </p>
+      <p>
+        For visualisation purposes, edges were pruned using the{" "}
+        <a target="_blank" rel="noreferrer" href="https://github.com/naviddianati/GraphPruning">
+          Marginal Likelihood Filter
+        </a>
+        .
+      </p>
+    </Panel>
+  );
+};
+
+export default DescriptionPanel;

src/views/GraphDataController.tsx

@@ -0,0 +1,45 @@
+import { useSigma } from "react-sigma-v2";
+import { FC, useEffect } from "react";
+import { keyBy, omit } from "lodash";
+
+import { Dataset, FiltersState } from "../types";
+
+const GraphDataController: FC<{ dataset: Dataset; filters: FiltersState }> = ({ dataset, filters, children }) => {
+  const sigma = useSigma();
+  const graph = sigma.getGraph();
+
+  /**
+   * Feed graphology with the new dataset:
+   */
+  useEffect(() => {
+    if (!graph || !dataset) return;
+
+    sigma.setCustomBBox(dataset.bbox)
+
+    const clusters = keyBy(dataset.clusters, "key");
+
+    dataset.nodes.forEach((node) =>
+      graph.addNode(node.key, {
+        ...node,
+        ...omit(clusters[node.cluster], "key"),
+      }),
+    );
+    dataset.edges.forEach(([source, target]) => graph.addEdge(source, target, { size: 1 }));
+
+    return () => graph.clear();
+  }, [graph, dataset]);
+
+  /**
+   * Apply filters to graphology:
+   */
+  useEffect(() => {
+    const { clusters } = filters;
+    graph.forEachNode((node, { cluster }) =>
+      graph.setNodeAttribute(node, "hidden", !clusters[cluster]),
+    );
+  }, [graph, filters]);
+
+  return <>{children}</>;
+};
+
+export default GraphDataController;

src/views/GraphEventsController.tsx

@@ -0,0 +1,35 @@
+import { useRegisterEvents, useSigma } from "react-sigma-v2";
+import { FC, useEffect } from "react";
+
+function getMouseLayer() {
+  return document.querySelector(".sigma-mouse");
+}
+
+const GraphEventsController: FC<{ setHoveredNode: (node: string | null) => void }> = ({ setHoveredNode, children }) => {
+  const registerEvents = useRegisterEvents();
+
+  /**
+   * Initialize here settings that require to know the graph and/or the sigma
+   * instance:
+   */
+  useEffect(() => {
+    registerEvents({
+      enterNode({ node }) {
+        setHoveredNode(node);
+        // TODO: Find a better way to get the DOM mouse layer:
+        const mouseLayer = getMouseLayer();
+        if (mouseLayer) mouseLayer.classList.add("mouse-pointer");
+      },
+      leaveNode() {
+        setHoveredNode(null);
+        // TODO: Find a better way to get the DOM mouse layer:
+        const mouseLayer = getMouseLayer();
+        if (mouseLayer) mouseLayer.classList.remove("mouse-pointer");
+      },
+    });
+  }, []);
+
+  return <>{children}</>;
+};
+
+export default GraphEventsController;

src/views/GraphSettingsController.tsx

@@ -0,0 +1,60 @@
+import { useSigma } from "react-sigma-v2";
+import { FC, useEffect } from "react";
+
+import { drawHover } from "../canvas-utils";
+import useDebounce from "../use-debounce";
+
+const NODE_FADE_COLOR = "#bbb";
+const EDGE_FADE_COLOR = "#eee";
+
+const GraphSettingsController: FC<{ hoveredNode: string | null }> = ({ children, hoveredNode }) => {
+  const sigma = useSigma();
+  const graph = sigma.getGraph();
+
+  // Here we debounce the value to avoid having too much highlights refresh when
+  // moving the mouse over the graph:
+  const debouncedHoveredNode = useDebounce(hoveredNode, 40);
+
+  /**
+   * Initialize here settings that require to know the graph and/or the sigma
+   * instance:
+   */
+  useEffect(() => {
+    sigma.setSetting("hoverRenderer", (context, data, settings) =>
+      drawHover(context, { ...sigma.getNodeDisplayData(data.key), ...data }, settings),
+    );
+  }, [sigma, graph]);
+
+  /**
+   * Update node and edge reducers when a node is hovered, to highlight its
+   * neighborhood:
+   */
+  useEffect(() => {
+    const hoveredColor: string = debouncedHoveredNode ? sigma.getNodeDisplayData(debouncedHoveredNode)!.color : "";
+
+    sigma.setSetting(
+      "nodeReducer",
+      debouncedHoveredNode
+        ? (node, data) =>
+            node === debouncedHoveredNode ||
+            graph.hasEdge(node, debouncedHoveredNode) ||
+            graph.hasEdge(debouncedHoveredNode, node)
+              ? { ...data, zIndex: 1 }
+              : { ...data, zIndex: 0, label: "", color: NODE_FADE_COLOR, image: null, highlighted: false }
+        : null,
+    );
+    sigma.setSetting(
+      "edgeReducer",
+      debouncedHoveredNode
+        ? (edge, data) =>
+            graph.hasExtremity(edge, debouncedHoveredNode)
+              ? { ...data, color: hoveredColor, size: 4 }
+              : { ...data, color: EDGE_FADE_COLOR, hidden: true }
+        : null,
+    );
+  }, [debouncedHoveredNode]);
+
+  return <>{children}</>;
+};
+
+export default GraphSettingsController;

src/views/GraphTitle.tsx

@@ -0,0 +1,47 @@
+import React, { FC, useEffect, useState } from "react";
+import { useSigma } from "react-sigma-v2";
+
+import { FiltersState } from "../types";
+
+function prettyPercentage(val: number): string {
+  return (val * 100).toFixed(1) + "%";
+}
+
+const GraphTitle: FC<{ filters: FiltersState }> = ({ filters }) => {
+  const sigma = useSigma();
+  const graph = sigma.getGraph();
+
+  const [visibleItems, setVisibleItems] = useState<{ nodes: number; edges: number }>({ nodes: 0, edges: 0 });
+  useEffect(() => {
+    // To ensure the graphology instance has up to data "hidden" values for
+    // nodes, we wait for next frame before reindexing. This won't matter in the
+    // UX, because of the visible nodes bar width transition.
+    requestAnimationFrame(() => {
+      const index = { nodes: 0, edges: 0 };
+      graph.forEachNode((_, { hidden }) => !hidden && index.nodes++);
+      graph.forEachEdge((_, _2, _3, _4, source, target) => !source.hidden && !target.hidden && index.edges++);
+      setVisibleItems(index);
+    });
+  }, [filters]);
+
+  return (
+    <div className="graph-title">
+      <h1>Entities mentioned in English-language posts from GESARA channels</h1>
+      <h2>
+        <i>
+          {graph.order} node{graph.order > 1 ? "s" : ""}{" "}
+          {visibleItems.nodes !== graph.order
+            ? ` (only ${prettyPercentage(visibleItems.nodes / graph.order)} visible)`
+            : ""}
+          , {graph.size} edge
+          {graph.size > 1 ? "s" : ""}{" "}
+          {visibleItems.edges !== graph.size
+            ? ` (only ${prettyPercentage(visibleItems.edges / graph.size)} visible)`
+            : ""}
+        </i>
+      </h2>
+    </div>
+  );
+};
+
+export default GraphTitle;

src/views/Panel.tsx

@@ -0,0 +1,37 @@
+import React, { FC, useEffect, useRef, useState } from "react";
+import { MdExpandLess, MdExpandMore } from "react-icons/md";
+import AnimateHeight from "react-animate-height";
+
+const DURATION = 300;
+
+const Panel: FC<{ title: JSX.Element | string; initiallyDeployed?: boolean }> = ({
+  title,
+  initiallyDeployed,
+  children,
+}) => {
+  const [isDeployed, setIsDeployed] = useState(initiallyDeployed || false);
+  const dom = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (isDeployed)
+      setTimeout(() => {
+        if (dom.current) dom.current.parentElement!.scrollTo({ top: dom.current.offsetTop - 5, behavior: "smooth" });
+      }, DURATION);
+  }, [isDeployed]);
+
+  return (
+    <div className="panel" ref={dom}>
+      <h2>
+        {title}{" "}
+        <button type="button" onClick={() => setIsDeployed((v) => !v)}>
+          {isDeployed ? <MdExpandLess /> : <MdExpandMore />}
+        </button>
+      </h2>
+      <AnimateHeight duration={DURATION} height={isDeployed ? "auto" : 0}>
+        {children}
+      </AnimateHeight>
+    </div>
+  );
+};
+
+export default Panel;

src/views/Root.tsx

@@ -0,0 +1,133 @@
+import React, { FC, useEffect, useState } from "react";
+import { SigmaContainer, ZoomControl, FullScreenControl } from "react-sigma-v2";
+import { omit, mapValues, keyBy, constant } from "lodash";
+
+import getNodeProgramImage from "sigma/rendering/webgl/programs/node.image";
+
+import GraphSettingsController from "./GraphSettingsController";
+import GraphEventsController from "./GraphEventsController";
+import GraphDataController from "./GraphDataController";
+import DescriptionPanel from "./DescriptionPanel";
+import { Dataset, FiltersState } from "../types";
+import ClustersPanel from "./ClustersPanel";
+import SearchField from "./SearchField";
+import drawLabel from "../canvas-utils";
+import GraphTitle from "./GraphTitle";
+
+import "react-sigma-v2/lib/react-sigma-v2.css";
+import { GrClose } from "react-icons/gr";
+import { BiRadioCircleMarked, BiBookContent } from "react-icons/bi";
+import { BsArrowsFullscreen, BsFullscreenExit, BsZoomIn, BsZoomOut } from "react-icons/bs";
+
+const Root: FC = () => {
+  const [showContents, setShowContents] = useState(false);
+  const [dataReady, setDataReady] = useState(false);
+  const [dataset, setDataset] = useState<Dataset | null>(null);
+  const [filtersState, setFiltersState] = useState<FiltersState>({
+    clusters: {},
+  });
+  const [hoveredNode, setHoveredNode] = useState<string | null>(null);
+
+  // Load data on mount:
+  useEffect(() => {
+    fetch(`${process.env.PUBLIC_URL}/dataset_entities.json`)
+      .then((res) => res.json())
+      .then((dataset: Dataset) => {
+        setDataset(dataset);
+        setFiltersState({
+          clusters: mapValues(keyBy(dataset.clusters, "key"), constant(true)),
+        });
+        requestAnimationFrame(() => setDataReady(true));
+      });
+  }, []);
+
+  if (!dataset) return null;
+
+  return (
+    <div id="app-root" className={showContents ? "show-contents" : ""}>
+      <SigmaContainer
+        graphOptions={{ type: "undirected" }}
+        initialSettings={{
+          nodeProgramClasses: { image: getNodeProgramImage() },
+          labelRenderer: drawLabel,
+          defaultNodeType: "image",
+          labelDensity: 0.07,
+          labelGridCellSize: 60,
+          labelRenderedSizeThreshold: 10,
+          labelFont: "Lato, sans-serif",
+          zIndex: true,
+        }}
+        className="react-sigma"
+      >
+        <GraphSettingsController hoveredNode={hoveredNode} />
+        <GraphEventsController setHoveredNode={setHoveredNode} />
+        <GraphDataController dataset={dataset} filters={filtersState} />
+
+        {dataReady && (
+          <>
+            <div className="controls">
+              <div className="ico">
+                <button
+                  type="button"
+                  className="show-contents"
+                  onClick={() => setShowContents(true)}
+                  title="Show caption and description"
+                >
+                  <BiBookContent />
+                </button>
+              </div>
+              <FullScreenControl
+                className="ico"
+                customEnterFullScreen={<BsArrowsFullscreen />}
+                customExitFullScreen={<BsFullscreenExit />}
+              />
+              <ZoomControl
+                className="ico"
+                customZoomIn={<BsZoomIn />}
+                customZoomOut={<BsZoomOut />}
+                customZoomCenter={<BiRadioCircleMarked />}
+              />
+            </div>
+            <div className="contents">
+              <div className="ico">
+                <button
+                  type="button"
+                  className="ico hide-contents"
+                  onClick={() => setShowContents(false)}
+                  title="Show caption and description"
+                >
+                  <GrClose />
+                </button>
+              </div>
+              <GraphTitle filters={filtersState} />
+              <div className="panels">
+                <SearchField filters={filtersState} />
+                <DescriptionPanel />
+                <ClustersPanel
+                  clusters={dataset.clusters}
+                  filters={filtersState}
+                  setClusters={(clusters) =>
+                    setFiltersState((filters) => ({
+                      ...filters,
+                      clusters,
+                    }))
+                  }
+                  toggleCluster={(cluster) => {
+                    setFiltersState((filters) => ({
+                      ...filters,
+                      clusters: filters.clusters[cluster]
+                        ? omit(filters.clusters, cluster)
+                        : { ...filters.clusters, [cluster]: true },
+                    }));
+                  }}
+                />
+              </div>
+            </div>
+          </>
+        )}
+      </SigmaContainer>
+    </div>
+  );
+};
+
+export default Root;

src/views/SearchField.tsx

@@ -0,0 +1,102 @@
+import React, { KeyboardEvent, ChangeEvent, FC, useEffect, useState } from "react";
+import { useSigma } from "react-sigma-v2";
+import { Attributes } from "graphology-types";
+import { BsSearch } from "react-icons/bs";
+
+import { FiltersState } from "../types";
+
+/**
+ * This component is basically a fork from React-sigma-v2's SearchControl
+ * component, to get some minor adjustments:
+ * 1. We need to hide hidden nodes from results
+ * 2. We need custom markup
+ */
+const SearchField: FC<{ filters: FiltersState }> = ({ filters }) => {
+  const sigma = useSigma();
+
+  const [search, setSearch] = useState<string>("");
+  const [values, setValues] = useState<Array<{ id: string; label: string }>>([]);
+  const [selected, setSelected] = useState<string | null>(null);
+
+  const refreshValues = () => {
+    const newValues: Array<{ id: string; label: string }> = [];
+    const lcSearch = search.toLowerCase();
+    if (!selected && search.length > 1) {
+      sigma.getGraph().forEachNode((key: string, attributes: Attributes): void => {
+        if (!attributes.hidden && attributes.label && attributes.label.toLowerCase().indexOf(lcSearch) === 0)
+          newValues.push({ id: key, label: attributes.label });
+      });
+    }
+    setValues(newValues);
+  };
+
+  // Refresh values when search is updated:
+  useEffect(() => refreshValues(), [search]);
+
+  // Refresh values when filters are updated (but wait a frame first):
+  useEffect(() => {
+    requestAnimationFrame(refreshValues);
+  }, [filters]);
+
+  useEffect(() => {
+    if (!selected) return;
+
+    sigma.getGraph().setNodeAttribute(selected, "highlighted", true);
+    const nodeDisplayData = sigma.getNodeDisplayData(selected);
+
+    if (nodeDisplayData)
+      sigma.getCamera().animate(
+        { ...nodeDisplayData, ratio: 0.05 },
+        {
+          duration: 600,
+        },
+      );
+
+    return () => {
+      sigma.getGraph().setNodeAttribute(selected, "highlighted", false);
+    };
+  }, [selected]);
+
+  const onInputChange = (e: ChangeEvent<HTMLInputElement>) => {
+    const searchString = e.target.value;
+    const valueItem = values.find((value) => value.label === searchString);
+    if (valueItem) {
+      setSearch(valueItem.label);
+      setValues([]);
+      setSelected(valueItem.id);
+    } else {
+      setSelected(null);
+      setSearch(searchString);
+    }
+  };
+
+  const onKeyPress = (e: KeyboardEvent<HTMLInputElement>) => {
+    if (e.key === "Enter" && values.length) {
+      setSearch(values[0].label);
+      setSelected(values[0].id);
+    }
+  };
+
+  return (
+    <div className="search-wrapper">
+      <input
+        type="search"
+        placeholder="Search in nodes..."
+        list="nodes"
+        value={search}
+        onChange={onInputChange}
+        onKeyPress={onKeyPress}
+      />
+      <BsSearch className="icon" />
+      <datalist id="nodes">
+        {values.map((value: { id: string; label: string }) => (
+          <option key={value.id} value={value.label}>
+            {value.label}
+          </option>
+        ))}
+      </datalist>
+    </div>
+  );
+};
+
+export default SearchField;