Typesense Search

Overview

Typesense is a fast, typo-tolerant search engine that powers the global search functionality in the Admin Frontend. This document covers the complete setup, configuration, indexing strategy, and API integration for high-performance full-text search across digital assets, folders, and collages.

Architecture

Components

1. Typesense Server: Search engine server (self-hosted or cloud)
2. Collections: Indexed data structures (digital_assets, digital_assets_categories, dam_collections)
3. Indexing Service: Backend service that syncs data to Typesense
4. Search API: Express.js endpoints for search operations
5. Frontend Client: Typesense JS client for browser-side search

Typesense Setup

Server Configuration

// typesense.config.js
module.exports = {
  server: {
    apiKey: process.env.TYPESENSE_API_KEY,
    nodes: [
      {
        host: process.env.TYPESENSE_HOST || 'localhost',
        port: process.env.TYPESENSE_PORT || 8108,
        protocol: process.env.TYPESENSE_PROTOCOL || 'http'
      }
    ],
    connectionTimeoutSeconds: 2
  }
}

Collection Schemas

Digital Assets Collection

{
  name: 'digital_assets',
  fields: [
    { name: 'id', type: 'string' },
    { name: 'workspace_id', type: 'int32', facet: true },
    { name: 'display_file_name', type: 'string' },
    { name: 'search_name', type: 'string' }, // Lowercase, normalized
    { name: 'description', type: 'string', optional: true },
    { name: 'file_type', type: 'string', facet: true },
    { name: 'file_size', type: 'int64' },
    { name: 'width', type: 'int32', optional: true },
    { name: 'height', type: 'int32', optional: true },
    { name: 'orientation', type: 'string', facet: true }, // horizontal, vertical, square
    { name: 'tags', type: 'string[]', facet: true },
    { name: 'category_id', type: 'int32', optional: true, facet: true },
    { name: 'uploaded_by', type: 'int32', facet: true },
    { name: 'uploaded_by_name', type: 'string' },
    { name: 'visibility', type: 'int32', facet: true }, // 0 = public, 1+ = private
    { name: 'created_at', type: 'int64' },
    { name: 'modified_at', type: 'int64' },
    { name: 'custom_fields', type: 'object', optional: true },
    { name: 's3_key', type: 'string' },
    { name: 'thumbnail_url', type: 'string', optional: true }
  ],
  default_sorting_field: 'modified_at'
}

Folders Collection

{
  name: 'digital_assets_categories',
  fields: [
    { name: 'id', type: 'string' },
    { name: 'workspace_id', type: 'int32', facet: true },
    { name: 'name', type: 'string' },
    { name: 'search_name', type: 'string' },
    { name: 'description', type: 'string', optional: true },
    { name: 'parent_id', type: 'int32', optional: true },
    { name: 'total_assets', type: 'int32' },
    { name: 'total_subfolders', type: 'int32' },
    { name: 'created_by', type: 'int32', facet: true },
    { name: 'created_by_name', type: 'string' },
    { name: 'visibility', type: 'int32', facet: true },
    { name: 'created_at', type: 'int64' },
    { name: 'modified_at', type: 'int64' }
  ],
  default_sorting_field: 'modified_at'
}

Collages Collection

{
  name: 'dam_collections',
  fields: [
    { name: 'id', type: 'string' },
    { name: 'workspace_id', type: 'int32', facet: true },
    { name: 'name', type: 'string' },
    { name: 'search_name', type: 'string' },
    { name: 'description', type: 'string', optional: true },
    { name: 'total_assets', type: 'int32' },
    { name: 'created_by', type: 'int32', facet: true },
    { name: 'created_by_name', type: 'string' },
    { name: 'visibility', type: 'int32', facet: true },
    { name: 'created_at', type: 'int64' },
    { name: 'modified_at', type: 'int64' }
  ],
  default_sorting_field: 'modified_at'
}

Backend Implementation

Typesense Client Service

// services/typesense.js
const Typesense = require('typesense')

class TypesenseService {
  constructor() {
    this.client = new Typesense.Client({
      nodes: [
        {
          host: process.env.TYPESENSE_HOST,
          port: process.env.TYPESENSE_PORT,
          protocol: process.env.TYPESENSE_PROTOCOL
        }
      ],
      apiKey: process.env.TYPESENSE_API_KEY,
      connectionTimeoutSeconds: 2
    })
  }

  async createCollections() {
    const collections = [
      this.getAssetsSchema(),
      this.getFoldersSchema(),
      this.getCollagesSchema()
    ]

    for (const schema of collections) {
      try {
        await this.client.collections().create(schema)
        console.log(`Created collection: ${schema.name}`)
      } catch (error) {
        if (error.httpStatus === 409) {
          console.log(`Collection ${schema.name} already exists`)
        } else {
          throw error
        }
      }
    }
  }

  async indexAsset(asset) {
    const document = {
      id: String(asset.id),
      workspace_id: asset.workspace_id,
      display_file_name: asset.display_file_name,
      search_name: asset.display_file_name.toLowerCase().trim(),
      description: asset.description || '',
      file_type: asset.file_type,
      file_size: asset.file_size,
      width: asset.width || null,
      height: asset.height || null,
      orientation: this.getOrientation(asset),
      tags: asset.tags?.map(t => t.name) || [],
      category_id: asset.category_id || null,
      uploaded_by: asset.uploaded_by,
      uploaded_by_name: asset.uploaded_by_name,
      visibility: asset.visibility || 0,
      created_at: Math.floor(new Date(asset.created_at).getTime() / 1000),
      modified_at: Math.floor(new Date(asset.modified_at).getTime() / 1000),
      custom_fields: this.formatCustomFields(asset.custom_fields),
      s3_key: asset.s3_key,
      thumbnail_url: asset.thumbnail_url || null
    }

    try {
      await this.client.collections('digital_assets').documents().upsert(document)
    } catch (error) {
      console.error('Error indexing asset:', error)
      throw error
    }
  }

  async indexFolder(folder) {
    const document = {
      id: String(folder.id),
      workspace_id: folder.workspace_id,
      name: folder.name,
      search_name: folder.name.toLowerCase().trim(),
      description: folder.description || '',
      parent_id: folder.parent_id || null,
      total_assets: folder.total_assets || 0,
      total_subfolders: folder.total_subfolders || 0,
      created_by: folder.created_by,
      created_by_name: folder.created_by_name,
      visibility: folder.visibility || 0,
      created_at: Math.floor(new Date(folder.created_at).getTime() / 1000),
      modified_at: Math.floor(new Date(folder.modified_at).getTime() / 1000)
    }

    await this.client.collections('digital_assets_categories').documents().upsert(document)
  }

  async indexCollage(collage) {
    const document = {
      id: String(collage.id),
      workspace_id: collage.workspace_id,
      name: collage.name,
      search_name: collage.name.toLowerCase().trim(),
      description: collage.description || '',
      total_assets: collage.total_assets || 0,
      created_by: collage.created_by,
      created_by_name: collage.created_by_name,
      visibility: collage.visibility || 0,
      created_at: Math.floor(new Date(collage.created_at).getTime() / 1000),
      modified_at: Math.floor(new Date(collage.modified_at).getTime() / 1000)
    }

    await this.client.collections('dam_collections').documents().upsert(document)
  }

  async deleteAsset(assetId) {
    await this.client.collections('digital_assets').documents(String(assetId)).delete()
  }

  async deleteFolder(folderId) {
    await this.client.collections('digital_assets_categories').documents(String(folderId)).delete()
  }

  async deleteCollage(collageId) {
    await this.client.collections('dam_collections').documents(String(collageId)).delete()
  }

  async search(request) {
    const { q, collections, filterQuery, commonSearchParams, workspace_id } = request

    const searches = collections.map(collection => ({
      collection,
      q: q || '*',
      query_by: commonSearchParams.query_by || 'search_name',
      filter_by: filterQuery[collection] || '',
      sort_by: commonSearchParams.sort_by || 'modified_at:desc',
      per_page: commonSearchParams.per_page || 10,
      page: commonSearchParams.page || 1,
      facet_by: commonSearchParams.facet_by || '',
      max_facet_values: commonSearchParams.max_facet_values || 10
    }))

    const results = await this.client.multiSearch.perform({ searches }, {})

    return results
  }

  getOrientation(asset) {
    if (!asset.width || !asset.height) return null
    if (asset.width > asset.height) return 'horizontal'
    if (asset.height > asset.width) return 'vertical'
    return 'square'
  }

  formatCustomFields(customFields) {
    if (!customFields) return {}
    const formatted = {}
    customFields.forEach(field => {
      formatted[field.field_key] = field.field_value
    })
    return formatted
  }
}

module.exports = new TypesenseService()

Search API Endpoint

// api/typesense.js
const express = require('express')
const router = express.Router()
const typesenseService = require('../services/typesense')
const { authenticate } = require('../middleware/auth')

router.post('/search', authenticate, async (req, res) => {
  try {
    const { request, filterQuery, commonSearchParams, workspace_id } = req.body

    // Validate workspace access
    if (!req.user.accessibleWorkspaces.includes(workspace_id)) {
      return res.status(403).json({ error: 'Access denied' })
    }

    // Add workspace filter to all collections
    const workspaceFilter = `workspace_id:=${workspace_id}`
    const enhancedFilterQuery = {}
    
    Object.keys(filterQuery).forEach(collection => {
      const existingFilter = filterQuery[collection]
      enhancedFilterQuery[collection] = existingFilter
        ? `(${workspaceFilter}) && (${existingFilter})`
        : workspaceFilter
    })

    const searchRequest = {
      q: request.q || '*',
      collections: request.collections || ['digital_assets', 'digital_assets_categories', 'dam_collections'],
      filterQuery: enhancedFilterQuery,
      commonSearchParams: {
        query_by: commonSearchParams?.query_by || 'search_name',
        per_page: commonSearchParams?.per_page || 20,
        page: commonSearchParams?.page || 1,
        sort_by: request.sort_by || 'modified_at:desc'
      },
      workspace_id
    }

    const results = await typesenseService.search(searchRequest)

    res.json({ results })
  } catch (error) {
    console.error('Search error:', error)
    res.status(500).json({ error: 'Search failed', message: error.message })
  }
})

module.exports = router

Frontend Implementation

Typesense Client Setup

// plugins/typesense.js
import Typesense from 'typesense'

export default ({ app, $axios }, inject) => {
  const typesenseClient = new Typesense.Client({
    nodes: [
      {
        host: process.env.TYPESENSE_HOST,
        port: process.env.TYPESENSE_PORT,
        protocol: process.env.TYPESENSE_PROTOCOL
      }
    ],
    apiKey: process.env.TYPESENSE_API_KEY,
    connectionTimeoutSeconds: 2
  })

  inject('typesense', typesenseClient)
}

Search Service

// utils/searchService.js
export class SearchService {
  constructor(axios, workspaceId) {
    this.axios = axios
    this.workspaceId = workspaceId
  }

  async search(query, options = {}) {
    const {
      collections = ['digital_assets', 'digital_assets_categories', 'dam_collections'],
      filters = {},
      sortBy = 'modified_at:desc',
      perPage = 20,
      page = 1
    } = options

    // Build filter query
    const filterQuery = {}
    collections.forEach(collection => {
      const collectionFilters = []
      
      // Visibility filter
      if (filters.visibility !== undefined) {
        collectionFilters.push(`visibility:=${filters.visibility}`)
      }
      
      // File type filter
      if (filters.fileTypes && collection === 'digital_assets') {
        const typeFilter = filters.fileTypes
          .map(type => `file_type:=${type}`)
          .join(' || ')
        collectionFilters.push(`(${typeFilter})`)
      }
      
      // Tags filter
      if (filters.tags && collection === 'digital_assets') {
        const tagFilter = filters.tags
          .map(tag => `tags:=[${tag}]`)
          .join(' && ')
        collectionFilters.push(`(${tagFilter})`)
      }
      
      // Date filter
      if (filters.dateFrom) {
        const timestamp = Math.floor(new Date(filters.dateFrom).getTime() / 1000)
        collectionFilters.push(`modified_at:>=${timestamp}`)
      }
      
      if (filters.dateTo) {
        const timestamp = Math.floor(new Date(filters.dateTo).getTime() / 1000)
        collectionFilters.push(`modified_at:<=${timestamp}`)
      }
      
      // Custom field filters
      if (filters.customFields) {
        Object.entries(filters.customFields).forEach(([key, value]) => {
          collectionFilters.push(`custom_fields.${key}:=${value}`)
        })
      }

      filterQuery[collection] = collectionFilters.length > 0
        ? `(${collectionFilters.join(' && ')})`
        : ''
    })

    const response = await this.axios.post(
      `/api/workspaces/${this.workspaceId}/typesense/search`,
      {
        request: {
          q: query || '*',
          collections,
          sort_by: sortBy
        },
        filterQuery,
        commonSearchParams: {
          query_by: 'search_name,description',
          per_page: perPage,
          page
        },
        workspace_id: this.workspaceId
      }
    )

    return this.transformResults(response.data.results)
  }

  transformResults(results) {
    const transformed = {
      assets: [],
      folders: [],
      collages: [],
      total: 0
    }

    results.forEach(result => {
      if (result.collection === 'digital_assets') {
        transformed.assets = result.hits.map(hit => ({
          ...hit.document,
          relevance_score: hit.text_match,
          highlights: hit.highlights
        }))
        transformed.total += result.found
      } else if (result.collection === 'digital_assets_categories') {
        transformed.folders = result.hits.map(hit => ({
          ...hit.document,
          relevance_score: hit.text_match
        }))
        transformed.total += result.found
      } else if (result.collection === 'dam_collections') {
        transformed.collages = result.hits.map(hit => ({
          ...hit.document,
          relevance_score: hit.text_match
        }))
        transformed.total += result.found
      }
    })

    // Sort by relevance score
    transformed.assets.sort((a, b) => b.relevance_score - a.relevance_score)
    transformed.folders.sort((a, b) => b.relevance_score - a.relevance_score)
    transformed.collages.sort((a, b) => b.relevance_score - a.relevance_score)

    return transformed
  }
}

Indexing Strategy

Real-time Indexing

Assets, folders, and collages are indexed immediately when created or updated:

// After asset upload
async function afterAssetUpload(asset) {
  // 1. Save asset to database
  const savedAsset = await db.assets.create(asset)
  
  // 2. Index in Typesense (async, non-blocking)
  typesenseService.indexAsset(savedAsset).catch(err => {
    console.error('Indexing error:', err)
    // Queue for retry
  })
  
  // 3. Generate thumbnail (async)
  generateThumbnail(savedAsset)
  
  return savedAsset
}

Batch Re-indexing

For bulk operations or initial setup:

// scripts/reindex.js
async function reindexAll(workspaceId) {
  const batchSize = 100
  
  // Re-index assets
  let offset = 0
  while (true) {
    const assets = await db.assets.findAll({
      where: { workspace_id: workspaceId },
      limit: batchSize,
      offset
    })
    
    if (assets.length === 0) break
    
    await Promise.all(
      assets.map(asset => typesenseService.indexAsset(asset))
    )
    
    offset += batchSize
    console.log(`Indexed ${offset} assets`)
  }
  
  // Similar for folders and collages
}

API Design

Multi-Collection Search

Endpoint: POST /api/workspaces/:workspace_id/typesense/search

Request Body:

{
  "request": {
    "q": "summer campaign",
    "collections": ["digital_assets", "digital_assets_categories", "dam_collections"],
    "sort_by": "modified_at:desc"
  },
  "filterQuery": {
    "digital_assets": "(file_type:=jpg || file_type:=png) && (tags:=[marketing])",
    "digital_assets_categories": "(visibility:=0)",
    "dam_collections": "(visibility:=0)"
  },
  "commonSearchParams": {
    "query_by": "search_name,description",
    "per_page": 20,
    "page": 1,
    "facet_by": "file_type,tags",
    "max_facet_values": 10
  },
  "workspace_id": "123456"
}

Response:

{
  "results": [
    {
      "collection": "digital_assets",
      "found": 45,
      "page": 1,
      "search_time_ms": 12,
      "hits": [
        {
          "document": {
            "id": "123",
            "display_file_name": "summer-campaign-banner.jpg",
            "file_type": "jpg",
            "tags": ["marketing", "summer"],
            "modified_at": 1739800483
          },
          "text_match": 95,
          "highlights": [
            {
              "field": "display_file_name",
              "snippet": "<mark>summer</mark>-<mark>campaign</mark>-banner.jpg"
            }
          ]
        }
      ],
      "facet_counts": [
        {
          "field_name": "file_type",
          "counts": [
            { "value": "jpg", "count": 25 },
            { "value": "png", "count": 20 }
          ]
        }
      ]
    }
  ]
}

Workflow

Search Execution Flow

1. User enters search query
   ↓
2. Frontend builds search request with filters
   ↓
3. Request sent to backend search API
   ↓
4. Backend validates workspace access
   ↓
5. Backend adds workspace filter to all collections
   ↓
6. Typesense multi-search executed
   ↓
7. Results returned from Typesense
   ↓
8. Backend transforms and merges results
   ↓
9. Results sorted by relevance (text_match score)
   ↓
10. Response sent to frontend
    ↓
11. Frontend displays results in tabs (All/Assets/Folders/Collages)

Indexing Flow

1. Asset/Folder/Collage created or updated
   ↓
2. Database record saved
   ↓
3. Indexing service triggered (async)
   ↓
4. Document formatted for Typesense
   ↓
5. Document upserted to Typesense collection
   ↓
6. Typesense indexes document
   ↓
7. Document immediately searchable

Sample Data

Typesense Document Examples

Asset Document

{
  id: "123",
  workspace_id: 456,
  display_file_name: "summer-campaign-banner.jpg",
  search_name: "summer-campaign-banner.jpg",
  description: "Main banner for summer marketing campaign",
  file_type: "jpg",
  file_size: 2456789,
  width: 1920,
  height: 1080,
  orientation: "horizontal",
  tags: ["marketing", "summer", "campaign", "banner"],
  category_id: 789,
  uploaded_by: 101,
  uploaded_by_name: "John Doe",
  visibility: 0,
  created_at: 1739800483,
  modified_at: 1739800483,
  custom_fields: {
    project_name: "Summer Campaign 2024",
    client_name: "Acme Corp",
    campaign_date: "2024-06-01"
  },
  s3_key: "456/digital_assets/123.jpg",
  thumbnail_url: "https://s3.amazonaws.com/bucket/456/thumbnails/123.jpg"
}

Folder Document

{
  id: "789",
  workspace_id: 456,
  name: "Marketing Assets",
  search_name: "marketing assets",
  description: "All marketing-related assets",
  parent_id: null,
  total_assets: 125,
  total_subfolders: 3,
  created_by: 101,
  created_by_name: "John Doe",
  visibility: 0,
  created_at: 1739800000,
  modified_at: 1739800483
}

Collage Document

{
  id: "321",
  workspace_id: 456,
  name: "Summer Campaign Collage",
  search_name: "summer campaign collage",
  description: "Collection of assets for summer campaign",
  total_assets: 15,
  created_by: 101,
  created_by_name: "John Doe",
  visibility: 0,
  created_at: 1739800200,
  modified_at: 1739800400
}

Related Documentation

• Global Search - Frontend search implementation
• Advanced Filters - Filtering system
• Tags and Custom Fields - Metadata system