import { useState, useCallback, useRef, useEffect, useMemo } from 'react'; import ReactFlow, { MiniMap, Controls, Background, useNodesState, useEdgesState, addEdge, Panel, useReactFlow, ReactFlowProvider, Handle, Position, BaseEdge, EdgeLabelRenderer, getBezierPath } from 'reactflow'; import 'reactflow/dist/style.css'; // Add custom CSS for React Flow const generateCustomStyles = () => { return ` .react-flow__node { z-index: 1; } .schema-background-node { z-index: -1; pointer-events: all; border-radius: 25px; cursor: grab; user-select: none; transition: all 0.2s ease; } .schema-background-node:hover { box-shadow: 0 0 30px rgba(0, 0, 0, 0.08); } .schema-background-node:active { cursor: grabbing; } .schema-background-node.dragging { opacity: 0.8; } .schema-label { position: absolute; top: 15px; left: 15px; background: rgba(255, 255, 255, 0.9); padding: 8px 12px; border-radius: 6px; font-size: 14px; font-weight: bold; box-shadow: 0 3px 6px rgba(0,0,0,0.1); display: flex; align-items: center; gap: 8px; z-index: 10; } `; }; // Import icons from react-icons import { FaDatabase, FaTable, FaArrowRight, FaExchangeAlt, FaLayerGroup } from 'react-icons/fa'; import { BiSolidData } from 'react-icons/bi'; import { AiFillFolder } from 'react-icons/ai'; import { BsFileEarmarkSpreadsheet } from 'react-icons/bs'; import { CustomProcessIcon } from './CustomIcons'; // Import mock data import mockApiData from './mockData'; // Import ProcessForm component import ProcessForm from './ProcessForm'; // Schema Background Node const SchemaBackgroundNode = ({ data }) => { // Different background colors for each schema const bgColor = data.slug === 'edw_schema' ? 'rgba(24, 144, 255, 0.1)' // Light blue for Schema_1 : 'rgba(82, 196, 26, 0.1)'; // Light green for Schema_2 // Border color based on schema const borderColor = data.slug === 'edw_schema' ? '#1890ff' // Blue for Schema_1 : '#52c41a'; // Green for Schema_2 return (
{data.name}
); }; // Custom edge with animated arrow const CustomEdge = ({ id, source, target, sourceX, sourceY, targetX, targetY, sourcePosition, targetPosition, style = {}, data, markerEnd }) => { const [edgePath, labelX, labelY] = getBezierPath({ sourceX, sourceY, sourcePosition, targetX, targetY, targetPosition, }); // Check if the process is active const isActive = data?.isActive !== false; // Default to active if not specified // Determine if this is a source or target edge const isSourceEdge = data?.edgeType === 'source'; // Get the process name and mappings const processName = data?.processName || 'Process'; const mappings = data?.mappings || []; // Adjust label style based on active status const labelBgColor = isActive ? 'white' : '#f5f5f5'; const labelOpacity = isActive ? 1 : 0.8; return ( <>
{/* Process name in small text */}
{processName} {!isActive && ( inactive )}
{/* Column mappings */}
{mappings.length > 0 ? ( mappings.map((column, index) => ( {column} )) ) : ( No mappings )}
); }; // Process Node (represents ETL or data transformation) const ProcessNode = ({ data, id }) => { // Function to handle process edit const handleProcessEdit = () => { // Find the process in the mock data const process = mockApiData.processes.find(p => p.slug === id); if (process) { // We'll use window.processEditCallback which will be set in the main component if (window.processEditCallback) { window.processEditCallback(process); } } }; // Determine status color const statusColor = data.status === 'active' ? '#52c41a' : '#ff4d4f'; const isActive = data.status === 'active'; // Define dimensions for the SVG const width = 240; const height = 140; const borderWidth = 2; // Define colors const primaryColor = isActive ? '#fa8c16' : '#aaaaaa'; const secondaryColor = isActive ? '#ff4d4f' : '#cccccc'; const bgColor = '#ffffff'; const descBgColor = isActive ? 'rgba(250, 140, 22, 0.08)' : 'rgba(170, 170, 170, 0.08)'; // Create gradient ID const gradientId = `process-gradient-${data.label.replace(/\s+/g, '-').toLowerCase()}`; return (
{/* SVG Container */} {/* Define gradient */} {/* Main ellipse shape */} {/* Description background */} {/* Connection handles */} {/* Content overlay */}
{/* Icon and title */}
{data.label}
{/* Status indicator */}
{data.status}
{/* Description */}
{data.description || 'Data transformation process'}
); }; // Table Node const TableNode = ({ data }) => { const isFact = data.type === 'fact'; // Use a consistent color theme for all tables const tableColor = '#1890ff'; // Blue for all tables const background = '#ffffff'; return (
{/* Connection handles */}
{/* Different icons for fact and dimension tables */} {isFact ? ( ) : ( )} {data.label} {isFact ? 'FACT' : 'DIM'}
{data.columns && (
Columns:
{data.columns.slice(0, 5).map((col, index) => ( {col} ))} {data.columns.length > 5 && ( +{data.columns.length - 5} more )}
)}
); }; // Main DataflowCanvas component const DataflowCanvas = () => { // State for nodes and edges const [showProcessForm, setShowProcessForm] = useState(false); const [selectedProcessForEdit, setSelectedProcessForEdit] = useState(null); const [showTableForm, setShowTableForm] = useState(false); const [selectedTableForEdit, setSelectedTableForEdit] = useState(null); // Get the React Flow instance const reactFlowInstance = useReactFlow(); // Define node types const nodeTypes = useMemo(() => ({ schemaBackground: SchemaBackgroundNode, process: ProcessNode, table: TableNode }), []); // Define edge types const edgeTypes = useMemo(() => ({ custom: CustomEdge }), []); // Create initial nodes from the mock data const initialNodes = useMemo(() => { // Create schema boundaries to properly size and position schema background nodes const schemaBoundaries = {}; // Initialize boundaries with schema positions mockApiData.schemas.forEach(schema => { schemaBoundaries[schema.slug] = { minX: schema.position.x, minY: schema.position.y, maxX: schema.position.x + schema.width, maxY: schema.position.y + schema.height }; }); // Update boundaries based on table positions mockApiData.tables.forEach(table => { const schemaSlug = table.schema; if (schemaBoundaries[schemaSlug]) { // Add some padding around tables (100px on each side) const tableMinX = table.orientation.x - 100; const tableMinY = table.orientation.y - 100; const tableMaxX = table.orientation.x + 280; // Table width + padding const tableMaxY = table.orientation.y + 280; // Table height + padding schemaBoundaries[schemaSlug].minX = Math.min(schemaBoundaries[schemaSlug].minX, tableMinX); schemaBoundaries[schemaSlug].minY = Math.min(schemaBoundaries[schemaSlug].minY, tableMinY); schemaBoundaries[schemaSlug].maxX = Math.max(schemaBoundaries[schemaSlug].maxX, tableMaxX); schemaBoundaries[schemaSlug].maxY = Math.max(schemaBoundaries[schemaSlug].maxY, tableMaxY); } }); // Schema background nodes (add these first so they appear behind other nodes) const schemaBackgroundNodes = mockApiData.schemas.map(schema => { const bounds = schemaBoundaries[schema.slug]; const width = bounds.maxX - bounds.minX; const height = bounds.maxY - bounds.minY; // Add 30% extra size const extraWidth = width * 0.3; const extraHeight = height * 0.3; return { id: `schema-bg-${schema.slug}`, type: 'schemaBackground', data: { name: schema.name, color: schema.color, slug: schema.slug }, position: { x: bounds.minX - (extraWidth / 2), y: bounds.minY - (extraHeight / 2) }, style: { width: width + extraWidth, height: height + extraHeight, zIndex: -1 // Ensure it's behind other nodes }, draggable: true, selectable: true, }; }); // Table nodes const tableNodes = mockApiData.tables.map(table => { return { id: table.slug, type: 'table', data: { label: table.name, type: table.type, columns: table.columns }, position: { x: table.orientation.x, y: table.orientation.y }, parentNode: `schema-bg-${table.schema}`, extent: 'parent', }; }); return [...schemaBackgroundNodes, ...tableNodes]; }, []); // Create process nodes const processNodes = useMemo(() => { return mockApiData.processes.map(process => { // Find source and destination tables to calculate position const sourceTables = mockApiData.tables.filter(table => process.source_table.includes(table.slug) ); const destTables = mockApiData.tables.filter(table => process.destination_table.includes(table.slug) ); // Calculate average position between source and destination tables let x = 0, y = 0, count = 0; let parentSchema = null; sourceTables.forEach(table => { x += table.orientation.x; y += table.orientation.y; count++; // If all tables are in the same schema, the process should be in that schema too if (!parentSchema) parentSchema = table.schema; else if (parentSchema !== table.schema) parentSchema = null; // Mixed schemas }); destTables.forEach(table => { x += table.orientation.x; y += table.orientation.y; count++; // If all tables are in the same schema, the process should be in that schema too if (!parentSchema) parentSchema = table.schema; else if (parentSchema !== table.schema) parentSchema = null; // Mixed schemas }); if (count > 0) { x = x / count; y = y / count; } // Position the process node between source and destination // For source tables on the left and dest tables on the right if (sourceTables.length > 0 && destTables.length > 0) { const avgSourceX = sourceTables.reduce((sum, t) => sum + t.orientation.x, 0) / sourceTables.length; const avgDestX = destTables.reduce((sum, t) => sum + t.orientation.x, 0) / destTables.length; // If source is to the left of destination, place in between if (avgSourceX < avgDestX) { x = avgSourceX + (avgDestX - avgSourceX) * 0.5; } // If source is to the right of destination, place in between else { x = avgDestX + (avgSourceX - avgDestX) * 0.5; } } return { id: process.slug, type: 'process', data: { label: process.name, description: process.description, type: process.type, status: process.status }, position: { x, y }, parentNode: parentSchema ? `schema-bg-${parentSchema}` : undefined, extent: parentSchema ? 'parent' : undefined, }; }); }, []); // Create edges between tables and processes const initialEdges = useMemo(() => { const edges = []; mockApiData.processes.forEach(process => { // Determine if process is active or inactive const isActive = process.status === 'active'; // Set colors and animation based on process status const sourceColor = isActive ? '#52c41a' : '#aaaaaa'; // Green or gray const destColor = isActive ? '#1890ff' : '#aaaaaa'; // Blue or gray const animated = isActive; // Only animate if active const strokeWidth = isActive ? 2 : 1.5; // Slightly thinner if inactive // Get the mappings for this process const mappings = process.mappings || []; // Format source columns for display const sourceColumns = mappings.map(m => m.source); // Format target columns for display const targetColumns = mappings.map(m => m.target); // Create edges from source tables to process process.source_table.forEach(sourceId => { edges.push({ id: `e-${sourceId}-${process.slug}`, source: sourceId, target: process.slug, type: 'custom', animated: animated, style: { stroke: sourceColor, strokeWidth: strokeWidth, opacity: isActive ? 1 : 0.7 // Slightly transparent if inactive }, markerEnd: { type: 'arrowclosed', width: 20, height: 20, color: sourceColor, }, data: { label: sourceColumns.join(', ') || 'No mappings', processName: process.name, isActive: isActive, // Pass status to the edge component edgeType: 'source', mappings: sourceColumns } }); }); // Create edges from process to destination tables process.destination_table.forEach(destId => { edges.push({ id: `e-${process.slug}-${destId}`, source: process.slug, target: destId, type: 'custom', animated: animated, style: { stroke: destColor, strokeWidth: strokeWidth, opacity: isActive ? 1 : 0.7 // Slightly transparent if inactive }, markerEnd: { type: 'arrowclosed', width: 20, height: 20, color: destColor, }, data: { label: targetColumns.join(', ') || 'No mappings', processName: process.name, isActive: isActive, // Pass status to the edge component edgeType: 'target', mappings: targetColumns } }); }); }); return edges; }, []); // Use React Flow's node state management const [nodes, setNodes, onNodesChangeDefault] = useNodesState([...initialNodes, ...processNodes]); const [edges, setEdges, onEdgesChange] = useEdgesState(initialEdges); // Custom nodes change handler to update schema boundaries when nodes move const onNodesChange = useCallback( (changes) => { // Apply the default node changes onNodesChangeDefault(changes); // Check if any of the changes are position changes const hasPositionChanges = changes.some( change => change.type === 'position' && change.dragging ); // If there are position changes, we might want to update schema boundaries // This is a placeholder for more complex logic if needed if (hasPositionChanges) { // console.log('Nodes are being moved'); } }, [onNodesChangeDefault] ); // Handle connecting nodes const onConnect = useCallback( (params) => { // Only allow connections from tables to processes or processes to tables const sourceNode = nodes.find(node => node.id === params.source); const targetNode = nodes.find(node => node.id === params.target); if (!sourceNode || !targetNode) return; // Check if connection is valid (table -> process or process -> table) const isSourceTable = sourceNode.type === 'table'; const isTargetProcess = targetNode.type === 'process'; const isSourceProcess = sourceNode.type === 'process'; const isTargetTable = targetNode.type === 'table'; if ((isSourceTable && isTargetProcess) || (isSourceProcess && isTargetTable)) { // Create a new edge with custom styling const newEdge = { ...params, type: 'custom', animated: true, style: { stroke: isSourceTable ? '#52c41a' : '#1890ff', strokeWidth: 2 }, markerEnd: { type: 'arrowclosed', width: 20, height: 20, color: isSourceTable ? '#52c41a' : '#1890ff', }, data: { label: 'New Connection', isActive: true } }; setEdges(eds => addEdge(newEdge, eds)); // TODO: Open a form to configure the connection // This would be where you define mappings, transformations, etc. } }, [nodes, setEdges] ); // Handle node click const onNodeClick = useCallback((event, node) => { // Prevent clicks on schema background nodes from opening forms if (node.type === 'schemaBackground') return; if (node.type === 'process') { // Find the process in the mock data const process = mockApiData.processes.find(p => p.slug === node.id); if (process) { setSelectedProcessForEdit(process); setShowProcessForm(true); } } else if (node.type === 'table') { // Find the table in the mock data const table = mockApiData.tables.find(t => t.slug === node.id); if (table) { setSelectedTableForEdit(table); setShowTableForm(true); } } }, []); // Add a new table const addTableNode = () => { const newTable = { id: `table-${Date.now()}`, type: 'table', data: { label: 'New Table', type: 'dimension', columns: ['id', 'name', 'description'] }, position: { x: Math.random() * 500, y: Math.random() * 500 }, }; setNodes(nodes => [...nodes, newTable]); }; // Get canvas boundaries for process placement const getSchemasBoundaries = () => { // Calculate based on existing nodes let minX = 100; let minY = 100; let maxX = 500; let maxY = 500; if (nodes.length > 0) { minX = Math.min(...nodes.map(node => node.position.x)); minY = Math.min(...nodes.map(node => node.position.y)); maxX = Math.max(...nodes.map(node => node.position.x + 200)); // Assuming node width is ~200px maxY = Math.max(...nodes.map(node => node.position.y + 200)); // Assuming node height is ~200px } return { minX, minY, maxX, maxY }; }; // We don't need the moveSchemaNodes function anymore // Open the process form to create a new process const addProcessNode = () => { setSelectedProcessForEdit(null); setShowProcessForm(true); }; // Handle saving a new process from the form const handleSaveProcess = (processData) => { const { minX, minY, maxX, maxY } = getSchemasBoundaries(); // Place the new process outside of all schemas // Either to the right of all schemas or below all schemas let newX, newY; // Get the current viewport to help position the new node in visible area const viewport = reactFlowInstance ? reactFlowInstance.getViewport() : { x: 0, y: 0, zoom: 1 }; // Determine placement position - right side is preferred newX = maxX + 200; newY = minY + (maxY - minY) / 2; // If we're editing an existing process, find it and update it if (selectedProcessForEdit) { const updatedNodes = nodes.map(node => { if (node.id === selectedProcessForEdit.slug) { return { ...node, data: { ...node.data, label: processData.name, description: processData.description, type: processData.type, status: processData.status, mappings: processData.mappings, filters: processData.filters, aggregations: processData.aggregations } }; } return node; }); setNodes(updatedNodes); // Update edges if source or destination tables have changed const oldProcess = mockApiData.processes.find(p => p.slug === selectedProcessForEdit.slug); // Remove old edges if (oldProcess) { const edgesToRemove = edges.filter(edge => edge.source === oldProcess.slug || edge.target === oldProcess.slug ); if (edgesToRemove.length > 0) { const remainingEdges = edges.filter(edge => !edgesToRemove.some(e => e.id === edge.id) ); setEdges(remainingEdges); } } // Create new edges const newEdges = []; // Determine if process is active or inactive const isActive = processData.status === 'active'; // Set colors and animation based on process status const sourceColor = isActive ? '#52c41a' : '#aaaaaa'; // Green or gray const destColor = isActive ? '#1890ff' : '#aaaaaa'; // Blue or gray const animated = isActive; // Only animate if active const strokeWidth = isActive ? 2 : 1.5; // Slightly thinner if inactive const opacity = isActive ? 1 : 0.7; // Slightly transparent if inactive // Get the mappings for this process const mappings = processData.mappings || []; // Format source columns for display const sourceColumns = mappings.map(m => m.source); // Format target columns for display const targetColumns = mappings.map(m => m.target); // Create edges from source tables to process processData.source_table.forEach(sourceId => { newEdges.push({ id: `e-${sourceId}-${processData.slug}`, source: sourceId, target: processData.slug, type: 'custom', animated: animated, style: { stroke: sourceColor, strokeWidth: strokeWidth, opacity: opacity }, markerEnd: { type: 'arrowclosed', width: 20, height: 20, color: sourceColor, }, data: { label: sourceColumns.join(', ') || 'No mappings', processName: processData.name, isActive: isActive, edgeType: 'source', mappings: sourceColumns } }); }); // Create edges from process to destination tables processData.destination_table.forEach(destId => { newEdges.push({ id: `e-${processData.slug}-${destId}`, source: processData.slug, target: destId, type: 'custom', animated: animated, style: { stroke: destColor, strokeWidth: strokeWidth, opacity: opacity }, markerEnd: { type: 'arrowclosed', width: 20, height: 20, color: destColor, }, data: { label: targetColumns.join(', ') || 'No mappings', processName: processData.name, isActive: isActive, edgeType: 'target', mappings: targetColumns } }); }); setEdges(eds => [...eds, ...newEdges]); // Update the mock data const processIndex = mockApiData.processes.findIndex(p => p.slug === processData.slug); if (processIndex !== -1) { mockApiData.processes[processIndex] = processData; } else { mockApiData.processes.push(processData); } } else { // Create a new process node const newProcess = { id: processData.slug, type: 'process', data: { label: processData.name, description: processData.description, type: processData.type, status: processData.status, mappings: processData.mappings, filters: processData.filters, aggregations: processData.aggregations }, position: { x: newX, y: newY } }; setNodes(nodes => [...nodes, newProcess]); // Create edges for the new process const newEdges = []; // Determine if process is active or inactive const isActive = processData.status === 'active'; // Set colors and animation based on process status const sourceColor = isActive ? '#52c41a' : '#aaaaaa'; // Green or gray const destColor = isActive ? '#1890ff' : '#aaaaaa'; // Blue or gray const animated = isActive; // Only animate if active const strokeWidth = isActive ? 2 : 1.5; // Slightly thinner if inactive // Get the mappings for this process const mappings = processData.mappings || []; // Format source columns for display const sourceColumns = mappings.map(m => m.source); // Format target columns for display const targetColumns = mappings.map(m => m.target); // Create edges from source tables to process processData.source_table.forEach(sourceId => { newEdges.push({ id: `e-${sourceId}-${processData.slug}`, source: sourceId, target: processData.slug, type: 'custom', animated: animated, style: { stroke: sourceColor, strokeWidth: strokeWidth, opacity: isActive ? 1 : 0.7 }, markerEnd: { type: 'arrowclosed', width: 20, height: 20, color: sourceColor, }, data: { label: sourceColumns.join(', ') || 'No mappings', processName: processData.name, isActive: isActive, edgeType: 'source', mappings: sourceColumns } }); }); // Create edges from process to destination tables processData.destination_table.forEach(destId => { newEdges.push({ id: `e-${processData.slug}-${destId}`, source: processData.slug, target: destId, type: 'custom', animated: animated, style: { stroke: destColor, strokeWidth: strokeWidth, opacity: isActive ? 1 : 0.7 }, markerEnd: { type: 'arrowclosed', width: 20, height: 20, color: destColor, }, data: { label: targetColumns.join(', ') || 'No mappings', processName: processData.name, isActive: isActive, edgeType: 'target', mappings: targetColumns } }); }); setEdges(eds => [...eds, ...newEdges]); // Add to mock data mockApiData.processes.push(processData); } }; // Add custom CSS to the document useEffect(() => { const styleElement = document.createElement('style'); styleElement.textContent = generateCustomStyles(); document.head.appendChild(styleElement); return () => { document.head.removeChild(styleElement); }; }, []); // Set initial viewport useEffect(() => { if (reactFlowInstance) { reactFlowInstance.setViewport(mockApiData.viewportSettings); } }, [reactFlowInstance]); // Set up a global callback for process editing // This is used by the ProcessNode component useEffect(() => { window.processEditCallback = (process) => { setSelectedProcessForEdit(process); setShowProcessForm(true); }; return () => { // Clean up when component unmounts window.processEditCallback = null; }; }, []); return (
{/* Process Form */} setShowProcessForm(false)} onSave={handleSaveProcess} tables={mockApiData.tables} existingProcess={selectedProcessForEdit} />
); }; // Wrap with ReactFlowProvider const DataflowCanvasWithProvider = () => ( ); export default DataflowCanvasWithProvider;