Chat sources, improve logger, UI improvements (#94)

core/api.py

@@ -64,8 +64,15 @@ async def readiness_check():
 # Initialize telemetry
 telemetry = TelemetryService()
 
-# Add OpenTelemetry instrumentation
+# Add OpenTelemetry instrumentation - exclude HTTP send/receive spans
-FastAPIInstrumentor.instrument_app(app)
+FastAPIInstrumentor.instrument_app(
+    app,
+    excluded_urls="health,health/.*",  # Exclude health check endpoints
+    exclude_spans=["send", "receive"],  # Exclude HTTP send/receive spans to reduce telemetry volume
+    http_capture_headers_server_request=None,  # Don't capture request headers
+    http_capture_headers_server_response=None,  # Don't capture response headers
+    tracer_provider=None  # Use the global tracer provider
+)
 
 # Add CORS middleware
 app.add_middleware(

core/database/postgres_database.py

@@ -750,16 +750,36 @@ class PostgresDatabase(BaseDatabase):
 
         filter_conditions = []
         for key, value in filters.items():
-            # Convert boolean values to string 'true' or 'false'
+            # Handle list of values (IN operator)
-            if isinstance(value, bool):
+            if isinstance(value, list):
-                value = str(value).lower()
+                if not value:  # Skip empty lists
+                    continue
                     
-            # Use proper SQL escaping for string values
+                # Build a list of properly escaped values
-            if isinstance(value, str):
+                escaped_values = []
-                # Replace single quotes with double single quotes to escape them
+                for item in value:
-                value = value.replace("'", "''") 
+                    if isinstance(item, bool):
+                        escaped_values.append(str(item).lower())
+                    elif isinstance(item, str):
+                        escaped_values.append(f"'{item.replace('\'', '\'\'')}'")
+                    else:
+                        escaped_values.append(f"'{item}'")
                         
-            filter_conditions.append(f"doc_metadata->>'{key}' = '{value}'")
+                # Join with commas for IN clause
+                values_str = ", ".join(escaped_values)
+                filter_conditions.append(f"doc_metadata->>'{key}' IN ({values_str})")
+            else:
+                # Handle single value (equality)
+                # Convert boolean values to string 'true' or 'false'
+                if isinstance(value, bool):
+                    value = str(value).lower()
+                    
+                # Use proper SQL escaping for string values
+                if isinstance(value, str):
+                    # Replace single quotes with double single quotes to escape them
+                    value = value.replace("'", "''") 
+                    
+                filter_conditions.append(f"doc_metadata->>'{key}' = '{value}'")
 
         return " AND ".join(filter_conditions)
         
@@ -773,12 +793,34 @@ class PostgresDatabase(BaseDatabase):
             if value is None:
                 continue
                 
-            if isinstance(value, str):
+            # Handle list of values (IN operator)
-                # Replace single quotes with double single quotes to escape them
+            if isinstance(value, list):
-                escaped_value = value.replace("'", "''")
+                if not value:  # Skip empty lists
-                conditions.append(f"system_metadata->>'{key}' = '{escaped_value}'")
+                    continue
+                    
+                # Build a list of properly escaped values
+                escaped_values = []
+                for item in value:
+                    if isinstance(item, bool):
+                        escaped_values.append(str(item).lower())
+                    elif isinstance(item, str):
+                        escaped_values.append(f"'{item.replace('\'', '\'\'')}'")
+                    else:
+                        escaped_values.append(f"'{item}'")
+                        
+                # Join with commas for IN clause
+                values_str = ", ".join(escaped_values)
+                conditions.append(f"system_metadata->>'{key}' IN ({values_str})")
             else:
-                conditions.append(f"system_metadata->>'{key}' = '{value}'")
+                # Handle single value (equality)
+                if isinstance(value, str):
+                    # Replace single quotes with double single quotes to escape them
+                    escaped_value = value.replace("'", "''")
+                    conditions.append(f"system_metadata->>'{key}' = '{escaped_value}'")
+                elif isinstance(value, bool):
+                    conditions.append(f"system_metadata->>'{key}' = '{str(value).lower()}'")
+                else:
+                    conditions.append(f"system_metadata->>'{key}' = '{value}'")
                 
         return " AND ".join(conditions)
 

core/services/document_service.py

@@ -1410,6 +1410,9 @@ class DocumentService:
         if metadata:
             doc.metadata.update(metadata)
             
+        # Ensure external_id is preserved in metadata
+        doc.metadata["external_id"] = doc.external_id
+        
         # Increment version
         current_version = doc.system_metadata.get("version", 1)
         doc.system_metadata["version"] = current_version + 1

core/workers/ingestion_worker.py

@@ -171,8 +171,13 @@ async def process_ingestion_job(
             raise ValueError(f"Document {document_id} not found in database after multiple retries")
             
         # Prepare updates for the document
+        # Merge new metadata with existing metadata to preserve external_id
+        merged_metadata = {**doc.metadata, **metadata}
+        # Make sure external_id is preserved in the metadata
+        merged_metadata["external_id"] = doc.external_id
+        
         updates = {
-            "metadata": metadata,
+            "metadata": merged_metadata,
             "additional_metadata": additional_metadata,
             "system_metadata": {**doc.system_metadata, "content": text}
         }

ui-component/app/page.tsx

@@ -7,8 +7,9 @@ import { useSearchParams } from 'next/navigation';
 function HomeContent() {
   const searchParams = useSearchParams();
   const folderParam = searchParams.get('folder');
+  const sectionParam = searchParams.get('section');
   
-  return <MorphikUI initialFolder={folderParam} />;
+  return <MorphikUI initialFolder={folderParam} initialSection={sectionParam || undefined} />;
 }
 
 export default function Home() {

ui-component/components/MorphikUI.tsx

@@ -22,7 +22,8 @@ const MorphikUI: React.FC<MorphikUIProps> = ({
   isReadOnlyUri = false, // Default to editable URI
   onUriChange,
   onBackClick,
-  initialFolder = null
+  initialFolder = null,
+  initialSection = 'documents'
 }) => {
   // State to manage connectionUri internally if needed
   const [currentUri, setCurrentUri] = useState(connectionUri);
@@ -40,7 +41,7 @@ const MorphikUI: React.FC<MorphikUIProps> = ({
       onUriChange(newUri);
     }
   };
-  const [activeSection, setActiveSection] = useState('documents');
+  const [activeSection, setActiveSection] = useState(initialSection);
   const [selectedGraphName, setSelectedGraphName] = useState<string | undefined>(undefined);
   const [isSidebarCollapsed, setIsSidebarCollapsed] = useState(false);
   

ui-component/components/chat/ChatMessage.tsx

@@ -1,14 +1,54 @@
 "use client";
 
 import React from 'react';
+import Image from 'next/image';
+import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion';
+import { Badge } from '@/components/ui/badge';
+import { Source } from '@/components/types';
 
 // Define our own props interface to avoid empty interface error
 interface ChatMessageProps {
   role: 'user' | 'assistant';
   content: string;
+  sources?: Source[];
 }
 
-const ChatMessageComponent: React.FC<ChatMessageProps> = ({ role, content }) => {
+const ChatMessageComponent: React.FC<ChatMessageProps> = ({ role, content, sources }) => {
+  // Helper to render content based on content type
+  const renderContent = (content: string, contentType: string) => {
+    if (contentType.startsWith('image/')) {
+      return (
+        <div className="flex justify-center p-4 bg-muted rounded-md">
+          <Image 
+            src={content} 
+            alt="Document content" 
+            className="max-w-full max-h-96 object-contain"
+            width={500}
+            height={300}
+          />
+        </div>
+      );
+    } else if (content.startsWith('data:image/png;base64,') || content.startsWith('data:image/jpeg;base64,')) {
+      return (
+        <div className="flex justify-center p-4 bg-muted rounded-md">
+          <Image 
+            src={content} 
+            alt="Base64 image content" 
+            className="max-w-full max-h-96 object-contain"
+            width={500}
+            height={300}
+          />
+        </div>
+      );
+    } else {
+      return (
+        <div className="bg-muted p-4 rounded-md whitespace-pre-wrap font-mono text-sm">
+          {content}
+        </div>
+      );
+    }
+  };
+
   return (
     <div className={`flex ${role === 'user' ? 'justify-end' : 'justify-start'}`}>
       <div 
@@ -19,6 +59,54 @@ const ChatMessageComponent: React.FC<ChatMessageProps> = ({ role, content }) =>
         }`}
       >
         <div className="whitespace-pre-wrap">{content}</div>
+        
+        {sources && sources.length > 0 && role === 'assistant' && (
+          <Accordion type="single" collapsible className="mt-4">
+            <AccordionItem value="sources">
+              <AccordionTrigger className="text-xs">Sources ({sources.length})</AccordionTrigger>
+              <AccordionContent>
+                <div className="space-y-2">
+                  {sources.map((source, index) => (
+                    <div key={`${source.document_id}-${source.chunk_number}-${index}`} className="bg-background p-2 rounded text-xs border">
+                      <div className="pb-2">
+                        <div className="flex justify-between items-start">
+                          <div>
+                            <span className="font-medium">
+                              {source.filename || `Document ${source.document_id.substring(0, 8)}...`}
+                            </span>
+                            <span className="text-muted-foreground ml-1">
+                              Chunk {source.chunk_number} {source.score !== undefined && `• Score: ${source.score.toFixed(2)}`}
+                            </span>
+                          </div>
+                          {source.content_type && (
+                            <Badge variant="outline" className="text-[10px]">
+                              {source.content_type}
+                            </Badge>
+                          )}
+                        </div>
+                      </div>
+                      
+                      {source.content && (
+                        renderContent(source.content, source.content_type || 'text/plain')
+                      )}
+                      
+                      <Accordion type="single" collapsible className="mt-3">
+                        <AccordionItem value="metadata">
+                          <AccordionTrigger className="text-[10px]">Metadata</AccordionTrigger>
+                          <AccordionContent>
+                            <pre className="bg-muted p-1 rounded text-[10px] overflow-x-auto">
+                              {JSON.stringify(source.metadata, null, 2)}
+                            </pre>
+                          </AccordionContent>
+                        </AccordionItem>
+                      </Accordion>
+                    </div>
+                  ))}
+                </div>
+              </AccordionContent>
+            </AccordionItem>
+          </Accordion>
+        )}
       </div>
     </div>
   );

ui-component/components/chat/ChatSection.tsx

@@ -10,7 +10,7 @@ import { showAlert } from '@/components/ui/alert-system';
 import ChatOptionsDialog from './ChatOptionsDialog';
 import ChatMessageComponent from './ChatMessage';
 
-import { ChatMessage, QueryOptions, Folder } from '@/components/types';
+import { ChatMessage, QueryOptions, Folder, Source } from '@/components/types';
 
 interface ChatSectionProps {
   apiBaseUrl: string;
@@ -34,6 +34,82 @@ const ChatSection: React.FC<ChatSectionProps> = ({ apiBaseUrl, authToken }) => {
     temperature: 0.7
   });
   
+  // Handle URL parameters for folder and filters
+  useEffect(() => {
+    if (typeof window !== 'undefined') {
+      const params = new URLSearchParams(window.location.search);
+      const folderParam = params.get('folder');
+      const filtersParam = params.get('filters');
+      const documentIdsParam = params.get('document_ids');
+      
+      let shouldShowChatOptions = false;
+      
+      // Update folder if provided
+      if (folderParam) {
+        try {
+          const folderName = decodeURIComponent(folderParam);
+          if (folderName) {
+            console.log(`Setting folder from URL parameter: ${folderName}`);
+            updateQueryOption('folder_name', folderName);
+            shouldShowChatOptions = true;
+          }
+        } catch (error) {
+          console.error('Error parsing folder parameter:', error);
+        }
+      }
+      
+      // Handle document_ids (selected documents) parameter - for backward compatibility
+      if (documentIdsParam) {
+        try {
+          const documentIdsJson = decodeURIComponent(documentIdsParam);
+          const documentIds = JSON.parse(documentIdsJson);
+          
+          // Create a filter object with external_id filter (correct field name)
+          const filtersObj = { external_id: documentIds };
+          const validFiltersJson = JSON.stringify(filtersObj);
+          
+          console.log(`Setting document_ids filter from URL parameter:`, filtersObj);
+          updateQueryOption('filters', validFiltersJson);
+          shouldShowChatOptions = true;
+        } catch (error) {
+          console.error('Error parsing document_ids parameter:', error);
+        }
+      }
+      // Handle general filters parameter
+      if (filtersParam) {
+        try {
+          const filtersJson = decodeURIComponent(filtersParam);
+          // Parse the JSON to confirm it's valid
+          const filtersObj = JSON.parse(filtersJson);
+          
+          console.log(`Setting filters from URL parameter:`, filtersObj);
+          
+          // Store the filters directly as a JSON string
+          updateQueryOption('filters', filtersJson);
+          shouldShowChatOptions = true;
+          
+          // Log a more helpful message about what's happening
+          if (filtersObj.external_id) {
+            console.log(`Chat will filter by ${Array.isArray(filtersObj.external_id) ? filtersObj.external_id.length : 1} document(s)`);
+          }
+        } catch (error) {
+          console.error('Error parsing filters parameter:', error);
+        }
+      }
+      
+      // Only show the chat options panel on initial parameter load
+      if (shouldShowChatOptions) {
+        setShowChatAdvanced(true);
+        
+        // Clear URL parameters after processing them to prevent modal from re-appearing on refresh
+        if (window.history.replaceState) {
+          const newUrl = window.location.pathname + window.location.hash;
+          window.history.replaceState({}, document.title, newUrl);
+        }
+      }
+    }
+  }, []);
+
   // Update query options
   const updateQueryOption = <K extends keyof QueryOptions>(key: K, value: QueryOptions[K]) => {
     setQueryOptions(prev => ({
@@ -141,8 +217,59 @@ const ChatSection: React.FC<ChatSectionProps> = ({ apiBaseUrl, authToken }) => {
       const data = await response.json();
       
       // Add assistant response to chat
-      const assistantMessage: ChatMessage = { role: 'assistant', content: data.completion };
+      const assistantMessage: ChatMessage = { 
+        role: 'assistant', 
+        content: data.completion,
+        sources: data.sources
+      };
       setChatMessages(prev => [...prev, assistantMessage]);
+      
+      // If sources are available, retrieve the full source content
+      if (data.sources && data.sources.length > 0) {
+        try {
+          // Fetch full source details
+          const sourcesResponse = await fetch(`${apiBaseUrl}/batch/chunks`, {
+            method: 'POST',
+            headers: {
+              'Authorization': authToken ? `Bearer ${authToken}` : '',
+              'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+              sources: data.sources,
+              folder_name: queryOptions.folder_name
+            })
+          });
+          
+          if (sourcesResponse.ok) {
+            const sourcesData = await sourcesResponse.json();
+            
+            // Process source data
+            
+            // Update the message with detailed source information
+            const updatedMessage = { 
+              ...assistantMessage, 
+              sources: sourcesData.map((source: Source) => ({
+                document_id: source.document_id,
+                chunk_number: source.chunk_number,
+                score: source.score,
+                content: source.content,
+                content_type: source.content_type || 'text/plain',
+                filename: source.filename,
+                metadata: source.metadata,
+                download_url: source.download_url
+              }))
+            };
+            
+            // Update the message with detailed sources
+            setChatMessages(prev => prev.map((msg, idx) => 
+              idx === prev.length - 1 ? updatedMessage : msg
+            ));
+          }
+        } catch (err) {
+          console.error('Error fetching source details:', err);
+          // Continue with basic sources if detailed fetch fails
+        }
+      }
       setChatQuery(''); // Clear input
     } catch (err) {
       const errorMsg = err instanceof Error ? err.message : 'An unknown error occurred';
@@ -173,6 +300,7 @@ const ChatSection: React.FC<ChatSectionProps> = ({ apiBaseUrl, authToken }) => {
                   key={index}
                   role={message.role}
                   content={message.content}
+                  sources={message.sources}
                 />
               ))}
             </div>

ui-component/components/documents/DocumentList.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import React, { useState } from 'react';
+import React, { useState, useEffect } from 'react';
 import { Checkbox } from "@/components/ui/checkbox";
 import { Badge } from '@/components/ui/badge';
 import { ScrollArea } from '@/components/ui/scroll-area';
@@ -8,7 +8,7 @@ import { Button } from '@/components/ui/button';
 import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog';
 import { Input } from '@/components/ui/input';
 import { Textarea } from '@/components/ui/textarea';
-import { Plus, Wand2, Upload } from 'lucide-react';
+import { Plus, Wand2, Upload, Filter } from 'lucide-react';
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { showAlert } from '@/components/ui/alert-system';
 
@@ -43,6 +43,76 @@ interface DocumentListProps {
   selectedFolder?: string | null;
 }
 
+// Filter Dialog Component
+const FilterDialog = ({
+  isOpen,
+  onClose,
+  columns,
+  filterValues,
+  setFilterValues
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+  columns: CustomColumn[];
+  filterValues: Record<string, string>;
+  setFilterValues: React.Dispatch<React.SetStateAction<Record<string, string>>>;
+}) => {
+  const [localFilters, setLocalFilters] = useState<Record<string, string>>(filterValues);
+
+  const handleApplyFilters = () => {
+    setFilterValues(localFilters);
+    onClose();
+  };
+
+  const handleClearFilters = () => {
+    setLocalFilters({});
+    setFilterValues({});
+    onClose();
+  };
+
+  const handleFilterChange = (column: string, value: string) => {
+    setLocalFilters(prev => ({
+      ...prev,
+      [column]: value
+    }));
+  };
+
+  return (
+    <Dialog open={isOpen} onOpenChange={(open) => !open && onClose()}>
+      <DialogContent onPointerDownOutside={(e) => e.preventDefault()}>
+        <DialogHeader>
+          <DialogTitle>Filter Documents</DialogTitle>
+          <DialogDescription>
+            Filter documents by their metadata values
+          </DialogDescription>
+        </DialogHeader>
+        <div className="space-y-4 py-4 max-h-96 overflow-y-auto">
+          {columns.map(column => (
+            <div key={column.name} className="space-y-2">
+              <label htmlFor={`filter-${column.name}`} className="text-sm font-medium">{column.name}</label>
+              <Input
+                id={`filter-${column.name}`}
+                placeholder={`Filter by ${column.name}...`}
+                value={localFilters[column.name] || ''}
+                onChange={(e) => handleFilterChange(column.name, e.target.value)}
+              />
+            </div>
+          ))}
+        </div>
+        <DialogFooter className="flex justify-between">
+          <Button variant="outline" onClick={handleClearFilters}>
+            Clear Filters
+          </Button>
+          <div className="flex gap-2">
+            <Button variant="outline" onClick={onClose}>Cancel</Button>
+            <Button onClick={handleApplyFilters}>Apply Filters</Button>
+          </div>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+};
+
 // Create a separate Column Dialog component to isolate its state
 const AddColumnDialog = ({ 
   isOpen, 
@@ -198,6 +268,9 @@ const DocumentList: React.FC<DocumentListProps> = ({
   const [customColumns, setCustomColumns] = useState<CustomColumn[]>([]);
   const [showAddColumnDialog, setShowAddColumnDialog] = useState(false);
   const [isExtracting, setIsExtracting] = useState(false);
+  const [showFilterDialog, setShowFilterDialog] = useState(false);
+  const [filterValues, setFilterValues] = useState<Record<string, string>>({});
+  const [filteredDocuments, setFilteredDocuments] = useState<Document[]>([]);
 
   // Get unique metadata fields from all documents
   const existingMetadataFields = React.useMemo(() => {
@@ -210,6 +283,29 @@ const DocumentList: React.FC<DocumentListProps> = ({
     return Array.from(fields);
   }, [documents]);
   
+  // Apply filter logic
+  useEffect(() => {
+    if (Object.keys(filterValues).length === 0) {
+      setFilteredDocuments(documents);
+      return;
+    }
+    
+    const filtered = documents.filter(doc => {
+      // Check if document matches all filter criteria
+      return Object.entries(filterValues).every(([key, value]) => {
+        if (!value || value.trim() === '') return true; // Skip empty filters
+        
+        const docValue = doc.metadata?.[key];
+        if (docValue === undefined) return false;
+        
+        // String comparison (case-insensitive)
+        return String(docValue).toLowerCase().includes(value.toLowerCase());
+      });
+    });
+    
+    setFilteredDocuments(filtered);
+  }, [documents, filterValues]);
+
   // Combine existing metadata fields with custom columns
   const allColumns = React.useMemo(() => {
     const metadataColumns: CustomColumn[] = existingMetadataFields.map(field => ({
@@ -237,6 +333,7 @@ const DocumentList: React.FC<DocumentListProps> = ({
   };
 
   // Handle data extraction
+
   const handleExtract = async () => {
     // First, find the folder object to get its ID
     if (!selectedFolder || customColumns.length === 0) {
@@ -408,46 +505,34 @@ const DocumentList: React.FC<DocumentListProps> = ({
     }
   };
 
-  const DocumentListHeader = () => (
+  // Calculate how many filters are currently active
-    <div className="bg-muted border-b font-medium sticky top-0 z-10 relative">
+  const activeFilterCount = Object.values(filterValues).filter(v => v && v.trim() !== '').length;
-      <div className="grid items-center w-full" style={{ 
+  
-        gridTemplateColumns: `48px minmax(200px, 350px) 100px 120px ${allColumns.map(() => '140px').join(' ')}` 
+  const DocumentListHeader = () => {
-      }}>
+    return (
-        <div className="flex items-center justify-center p-3">
+      <div className="bg-muted border-b font-medium sticky top-0 z-10 relative">
-          <Checkbox
+        <div className="grid items-center w-full" style={{ 
-            id="select-all-documents"
+          gridTemplateColumns: `48px minmax(200px, 350px) 100px 120px ${allColumns.map(() => '140px').join(' ')}` 
-            checked={getSelectAllState()}
+        }}>
-            onCheckedChange={(checked) => {
+          <div className="flex items-center justify-center p-3">
-              if (checked) {
+            <Checkbox
-                setSelectedDocuments(documents.map(doc => doc.external_id));
+              id="select-all-documents"
-              } else {
+              checked={getSelectAllState()}
-                setSelectedDocuments([]);
+              onCheckedChange={(checked) => {
-              }
+                if (checked) {
-            }}
+                  setSelectedDocuments(documents.map(doc => doc.external_id));
-            aria-label="Select all documents"
+                } else {
-          />
+                  setSelectedDocuments([]);
-        </div>
+                }
-        <div className="text-sm font-semibold p-3">Filename</div>
+              }}
-        <div className="text-sm font-semibold p-3">Type</div>
+              aria-label="Select all documents"
-        <div className="text-sm font-semibold p-3">
+            />
-          <div className="group relative inline-flex items-center">
-            Status
-            <span className="ml-1 text-muted-foreground cursor-help">
-              <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
-                <circle cx="12" cy="12" r="10"></circle>
-                <line x1="12" y1="16" x2="12" y2="12"></line>
-                <line x1="12" y1="8" x2="12.01" y2="8"></line>
-              </svg>
-            </span>
-            <div className="absolute left-0 top-6 hidden group-hover:block bg-background border text-foreground text-xs p-3 rounded-md w-64 z-[100] shadow-lg">
-              Documents with &quot;Processing&quot; status are queryable, but visual features like direct visual context will only be available after processing completes.
-            </div>
           </div>
-        </div>
+          <div className="text-sm font-semibold p-3">Filename</div>
-        {allColumns.map((column) => (
+          <div className="text-sm font-semibold p-3">Type</div>
-          <div key={column.name} className="text-sm font-semibold p-3">
+          <div className="text-sm font-semibold p-3">
             <div className="group relative inline-flex items-center">
-              {column.name}
+              Status
               <span className="ml-1 text-muted-foreground cursor-help">
                 <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
                   <circle cx="12" cy="12" r="10"></circle>
@@ -456,38 +541,50 @@ const DocumentList: React.FC<DocumentListProps> = ({
                 </svg>
               </span>
               <div className="absolute left-0 top-6 hidden group-hover:block bg-background border text-foreground text-xs p-3 rounded-md w-64 z-[100] shadow-lg">
-                <p>{column.description}</p>
+                Documents with &quot;Processing&quot; status are queryable, but visual features like direct visual context will only be available after processing completes.
-                <p className="mt-1 font-medium">Type: {column._type}</p>
-                {column.schema && (
-                  <p className="mt-1 text-xs">Schema provided</p>
-                )}
               </div>
             </div>
           </div>
-        ))}
+          {allColumns.map((column) => (
-      </div>
+            <div key={column.name} className="text-sm font-semibold p-3">
+              <div className="group relative inline-flex items-center">
+                {column.name}
+                <span className="ml-1 text-muted-foreground cursor-help">
+                  <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
+                    <circle cx="12" cy="12" r="10"></circle>
+                    <line x1="12" y1="16" x2="12" y2="12"></line>
+                    <line x1="12" y1="8" x2="12.01" y2="8"></line>
+                  </svg>
+                </span>
+                <div className="absolute left-0 top-6 hidden group-hover:block bg-background border text-foreground text-xs p-3 rounded-md w-64 z-[100] shadow-lg">
+                  <p>{column.description}</p>
+                  <p className="mt-1 font-medium">Type: {column._type}</p>
+                  {column.schema && (
+                    <p className="mt-1 text-xs">Schema provided</p>
+                  )}
+                </div>
+              </div>
+            </div>
+          ))}
+        </div>
         
-      <div className="absolute right-3 top-1/2 transform -translate-y-1/2">
+        {/* Render dialogs separately */}
-        <Button 
-          variant="ghost" 
-          size="icon" 
-          className="h-6 w-6 rounded-full" 
-          title="Add column"
-          onClick={() => setShowAddColumnDialog(true)}
-        >
-          <Plus className="h-4 w-4" />
-          <span className="sr-only">Add column</span>
-        </Button>
-        
-        {/* Render the dialog separately */}
         <AddColumnDialog 
           isOpen={showAddColumnDialog}
           onClose={() => setShowAddColumnDialog(false)}
           onAddColumn={handleAddColumn}
         />
+        
+        <FilterDialog
+          isOpen={showFilterDialog}
+          onClose={() => setShowFilterDialog(false)}
+          columns={allColumns}
+          filterValues={filterValues}
+          setFilterValues={setFilterValues}
+        />
       </div>
-    </div>
+    );
-  );
+  };
 
   if (loading && !documents.length) {
     return (
@@ -507,7 +604,7 @@ const DocumentList: React.FC<DocumentListProps> = ({
     <div className="border rounded-md overflow-hidden shadow-sm w-full">
       <DocumentListHeader />
       <ScrollArea className="h-[calc(100vh-220px)]">
-        {documents.map((doc) => (
+        {filteredDocuments.map((doc) => (
           <div 
             key={doc.external_id}
             onClick={() => handleDocumentClick(doc)}
@@ -569,6 +666,24 @@ const DocumentList: React.FC<DocumentListProps> = ({
           </div>
         ))}
         
+        {filteredDocuments.length === 0 && documents.length > 0 && (
+          <div className="p-12 text-center flex flex-col items-center justify-center">
+            <div className="h-12 w-12 rounded-full bg-muted flex items-center justify-center mb-4">
+              <Filter className="text-muted-foreground" />
+            </div>
+            <p className="text-muted-foreground">
+              No documents match the current filters.
+            </p>
+            <Button 
+              variant="link" 
+              className="mt-2" 
+              onClick={() => setFilterValues({})}
+            >
+              Clear all filters
+            </Button>
+          </div>
+        )}
+        
         {documents.length === 0 && (
           <div className="p-12 text-center flex flex-col items-center justify-center">
             <div className="h-12 w-12 rounded-full bg-muted flex items-center justify-center mb-4">
@@ -588,18 +703,66 @@ const DocumentList: React.FC<DocumentListProps> = ({
         )}
       </ScrollArea>
       
-      {customColumns.length > 0 && (
+      <div className="border-t p-3 flex justify-between">
-        <div className="border-t p-3 flex justify-end">
+        {/* Filter stats */}
-          <Button 
+        <div className="flex items-center text-sm text-muted-foreground">
-            className="gap-2" 
+          {Object.keys(filterValues).length > 0 ? (
-            onClick={handleExtract}
+            <div className="flex items-center gap-1">
-            disabled={isExtracting || !selectedFolder}
+              <Filter className="h-4 w-4" />
-          >
+              <span>
-            <Wand2 className="h-4 w-4" />
+                {filteredDocuments.length} of {documents.length} documents
-            {isExtracting ? 'Processing...' : 'Extract'}
+                {Object.keys(filterValues).length > 0 && (
-          </Button>
+                  <Button variant="link" className="p-0 h-auto text-sm ml-1" onClick={() => setFilterValues({})}>
+                    Clear filters
+                  </Button>
+                )}
+              </span>
+            </div>
+          ) : null}
         </div>
-      )}
+        
+        {/* Action buttons */}
+        <div className="flex gap-2">
+          {/* Filter button */}
+          <Button 
+            variant={activeFilterCount > 0 ? "default" : "outline"}
+            size="sm" 
+            className="h-8 text-xs font-medium"
+            onClick={() => setShowFilterDialog(true)}
+          >
+            <Filter className="h-3.5 w-3.5 mr-0.5" />
+            Filter
+            {activeFilterCount > 0 && (
+              <span className="ml-1 h-4 w-4 bg-primary/20 text-primary text-[10px] flex items-center justify-center rounded-full">
+                {activeFilterCount}
+              </span>
+            )}
+          </Button>
+          
+          {/* Add column button */}
+          <Button 
+            variant="outline"
+            size="sm"
+            className="h-8 text-xs font-medium"  
+            title="Add column"
+            onClick={() => setShowAddColumnDialog(true)}
+          >
+            <Plus className="h-3.5 w-3.5 mr-0.5" />
+            Column
+          </Button>
+          
+          {customColumns.length > 0 && selectedFolder && (
+            <Button 
+              className="gap-2" 
+              onClick={handleExtract}
+              disabled={isExtracting || !selectedFolder}
+            >
+              <Wand2 className="h-4 w-4" />
+              {isExtracting ? 'Processing...' : 'Extract'}
+            </Button>
+          )}
+        </div>
+      </div>
     </div>
   );
 };

ui-component/components/documents/DocumentsSection.tsx

@@ -1326,14 +1326,7 @@ const DocumentsSection: React.FC<DocumentsSectionProps> = ({
         />
       )}
       
-      {documents.length === 0 && !loading && folders.length === 0 && !foldersLoading ? (
+      {selectedFolder && documents.length === 0 && !loading ? (
-        <div className="text-center py-8 border border-dashed rounded-lg flex-1 flex items-center justify-center">
-          <div>
-            <Upload className="mx-auto h-12 w-12 mb-2 text-muted-foreground" />
-            <p className="text-muted-foreground">No documents found. Upload your first document.</p>
-          </div>
-        </div>
-      ) : selectedFolder && documents.length === 0 && !loading ? (
         <div className="text-center py-8 border border-dashed rounded-lg flex-1 flex items-center justify-center">
           <div>
             <Upload className="mx-auto h-12 w-12 mb-2 text-muted-foreground" />

ui-component/components/documents/FolderList.tsx

@@ -2,7 +2,7 @@
 
 import React from 'react';
 import { Button } from '@/components/ui/button';
-import { PlusCircle, ArrowLeft } from 'lucide-react';
+import { PlusCircle, ArrowLeft, MessageSquare } from 'lucide-react';
 import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog';
 import { Label } from '@/components/ui/label';
 import { Input } from '@/components/ui/input';
@@ -131,15 +131,51 @@ const FolderList: React.FC<FolderListProps> = ({
               </h2>
             </div>
             
-            {/* Show delete button if documents are selected */}
+            {/* Show action buttons if documents are selected */}
-            {selectedDocuments.length > 0 && handleDeleteMultipleDocuments && (
+            {selectedDocuments && selectedDocuments.length > 0 && (
-              <Button 
+              <div className="flex gap-2 ml-4">
-                variant="outline" 
+                {/* Chat with selected button */}
-                onClick={handleDeleteMultipleDocuments}
+                <Button 
-                className="border-red-500 text-red-500 hover:bg-red-50 ml-4"
+                  variant="outline" 
-              >
+                  onClick={() => {
-                Delete {selectedDocuments.length} selected
+                    // Build proper URL path
-              </Button>
+                    let path = '/';
+                    if (typeof window !== 'undefined') {
+                      const currentPath = window.location.pathname;
+                      if (currentPath !== '/') {
+                        path = currentPath;
+                      }
+                    }
+                    
+                    // Create filter with external_id which is the correct field name
+                    const filter = JSON.stringify({ external_id: selectedDocuments });
+                    const filtersParam = encodeURIComponent(filter);
+                    
+                    // Navigate to chat with selected documents
+                    // Use window.location to force a full page reload
+                    if (typeof window !== 'undefined') {
+                      window.location.href = `${path}?section=chat&filters=${filtersParam}`;
+                    } else {
+                      router.push(`${path}?section=chat&filters=${filtersParam}`);
+                    }
+                  }}
+                  className="border-primary text-primary hover:bg-primary/10 flex items-center gap-1"
+                >
+                  <MessageSquare size={16} />
+                  Chat with {selectedDocuments.length} selected
+                </Button>
+                
+                {/* Delete button */}
+                {handleDeleteMultipleDocuments && (
+                  <Button 
+                    variant="outline" 
+                    onClick={handleDeleteMultipleDocuments}
+                    className="border-red-500 text-red-500 hover:bg-red-50"
+                  >
+                    Delete {selectedDocuments.length} selected
+                  </Button>
+                )}
+              </div>
             )}
           </div>
           

ui-component/components/types.ts

@@ -8,6 +8,7 @@ export interface MorphikUIProps {
   onBackClick?: () => void; // Callback when back button is clicked
   appName?: string; // Name of the app to display in UI
   initialFolder?: string | null; // Initial folder to show
+  initialSection?: string; // Initial section to show (documents, search, chat, etc.)
 }
 
 export interface Document {
@@ -41,9 +42,21 @@ export interface SearchResult {
   metadata: Record<string, unknown>;
 }
 
+export interface Source {
+  document_id: string;
+  chunk_number: number;
+  score?: number;
+  filename?: string;
+  content?: string;
+  content_type?: string;
+  metadata?: Record<string, unknown>;
+  download_url?: string;
+}
+
 export interface ChatMessage {
   role: 'user' | 'assistant';
   content: string;
+  sources?: Source[];
 }
 
 export interface SearchOptions {