neo/backend/src/object/object.service.ts

import { Injectable, NotFoundException, Logger } from '@nestjs/common';
import { TenantDatabaseService } from '../tenant/tenant-database.service';
import { CustomMigrationService } from '../migration/custom-migration.service';
import { ModelService } from './models/model.service';
import { AuthorizationService } from '../rbac/authorization.service';
import { SchemaManagementService } from './schema-management.service';
import { ObjectDefinition } from '../models/object-definition.model';
import { FieldDefinition } from '../models/field-definition.model';
import { User } from '../models/user.model';
import { ObjectMetadata } from './models/dynamic-model.factory';

@Injectable()
export class ObjectService {
  private readonly logger = new Logger(ObjectService.name);
  
  constructor(
    private tenantDbService: TenantDatabaseService,
    private customMigrationService: CustomMigrationService,
    private modelService: ModelService,
    private authService: AuthorizationService,
  ) {}

  // Setup endpoints - Object metadata management
  async getObjectDefinitions(tenantId: string) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    const objects = await knex('object_definitions')
      .select('object_definitions.*')
      .orderBy('label', 'asc');

    // Fetch app information for objects that have app_id
    for (const obj of objects) {
      if (obj.app_id) {
        const app = await knex('apps')
          .where({ id: obj.app_id })
          .select('id', 'slug', 'label', 'description')
          .first();
        obj.app = app;
      }
    }

    return objects;
  }

  async getObjectDefinition(tenantId: string, apiName: string) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    const obj = await knex('object_definitions')
      .where({ apiName })
      .first();

    if (!obj) {
      throw new NotFoundException(`Object ${apiName} not found`);
    }

    // Get fields for this object
    const fields = await knex('field_definitions')
      .where({ objectDefinitionId: obj.id })
      .orderBy('label', 'asc');

    // Normalize all fields to ensure system fields are properly marked
    const normalizedFields = fields.map((field: any) => this.normalizeField(field));

    // Get app information if object belongs to an app
    let app = null;
    if (obj.app_id) {
      app = await knex('apps')
        .where({ id: obj.app_id })
        .select('id', 'slug', 'label', 'description')
        .first();
    }

    return {
      ...obj,
      fields: normalizedFields,
      app,
    };
  }

  async createObjectDefinition(
    tenantId: string,
    data: {
      apiName: string;
      label: string;
      pluralLabel?: string;
      description?: string;
      isSystem?: boolean;
    },
  ) {
    // Resolve tenant ID in case a slug was passed
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Generate UUID for the new object
    const objectId = require('crypto').randomUUID();
    
    // Create the object definition record
    await knex('object_definitions').insert({
      id: objectId,
      ...data,
      created_at: knex.fn.now(),
      updated_at: knex.fn.now(),
    });
    
    const objectDef = await knex('object_definitions').where({ id: objectId }).first();
    
    // Create standard field definitions (only if they don't already exist)
    const standardFields = [
      {
        apiName: 'ownerId',
        label: 'Owner',
        type: 'LOOKUP',
        description: 'The user who owns this record',
        isRequired: false,  // Auto-set by system
        isUnique: false,
        referenceObject: 'User',
        isSystem: true,
        isCustom: false,
      },
      {
        apiName: 'name',
        label: 'Name',
        type: 'STRING',
        description: 'The primary name field for this record',
        isRequired: false,  // Optional field
        isUnique: false,
        referenceObject: null,
        isSystem: false,
        isCustom: false,
      },
      {
        apiName: 'created_at',
        label: 'Created At',
        type: 'DATE_TIME',
        description: 'The timestamp when this record was created',
        isRequired: false,  // Auto-set by system
        isUnique: false,
        referenceObject: null,
        isSystem: true,
        isCustom: false,
      },
      {
        apiName: 'updated_at',
        label: 'Updated At',
        type: 'DATE_TIME',
        description: 'The timestamp when this record was last updated',
        isRequired: false,  // Auto-set by system
        isUnique: false,
        referenceObject: null,
        isSystem: true,
        isCustom: false,
      },
    ];

    // Insert standard field definitions that don't already exist
    for (const field of standardFields) {
      const existingField = await knex('field_definitions')
        .where({
          objectDefinitionId: objectDef.id,
          apiName: field.apiName,
        })
        .first();

      if (!existingField) {
        const fieldData: any = {
          id: knex.raw('(UUID())'),
          objectDefinitionId: objectDef.id,
          ...field,
          created_at: knex.fn.now(),
          updated_at: knex.fn.now(),
        };
        
        // For lookup fields, set ui_metadata with relationDisplayField
        if (field.type === 'LOOKUP') {
          fieldData.ui_metadata = JSON.stringify({
            relationDisplayField: 'name',
          });
        }
        
        await knex('field_definitions').insert(fieldData);
      }
    }
    
    // Create a migration to create the table
    const tableName = this.getTableName(data.apiName);
    const createTableSQL = this.customMigrationService.generateCreateTableSQL(tableName);
    
    try {
      await this.customMigrationService.createAndExecuteMigration(
        knex,
        resolvedTenantId,
        {
          name: `create_${tableName}_table`,
          description: `Create table for ${data.label} object`,
          type: 'create_table',
          sql: createTableSQL,
        },
      );
    } catch (error) {
      // Log the error but don't fail - migration is recorded for future retry
      console.error(`Failed to execute table creation migration: ${error.message}`);
    }

    // Create and register the Objection model for this object
    try {
      const allFields = await knex('field_definitions')
        .where({ objectDefinitionId: objectDef.id })
        .select('apiName', 'label', 'type', 'isRequired', 'isUnique', 'referenceObject');

      const objectMetadata: ObjectMetadata = {
        apiName: data.apiName,
        tableName,
        fields: allFields.map((f: any) => ({
          apiName: f.apiName,
          label: f.label,
          type: f.type,
          isRequired: f.isRequired,
          isUnique: f.isUnique,
          referenceObject: f.referenceObject,
        })),
        relations: [],
      };

      await this.modelService.createModelForObject(resolvedTenantId, objectMetadata);
    } catch (error) {
      console.error(`Failed to create model for object ${data.apiName}:`, error.message);
    }
    
    return objectDef;
  }

  async updateObjectDefinition(
    tenantId: string,
    objectApiName: string,
    data: Partial<{
      label: string;
      pluralLabel: string;
      description: string;
      orgWideDefault: 'private' | 'public_read' | 'public_read_write';
    }>,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Update the object definition
    await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName })
      .patch({
        ...data,
        updatedAt: new Date(),
      });
    
    // Return updated object
    return await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName });
  }

  async createFieldDefinition(
    tenantId: string,
    objectApiName: string,
    data: {
      apiName: string;
      label: string;
      type: string;
      description?: string;
      isRequired?: boolean;
      isUnique?: boolean;
      referenceObject?: string;
      relationObject?: string;
      relationDisplayField?: string;
      defaultValue?: string;
      length?: number;
      precision?: number;
      scale?: number;
      uiMetadata?: any;
    },
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    const obj = await this.getObjectDefinition(tenantId, objectApiName);

    // Convert frontend type to database type
    const dbFieldType = this.convertFrontendFieldType(data.type);
    
    // Use relationObject if provided (alias for referenceObject)
    const referenceObject = data.referenceObject || data.relationObject;

    // Generate UUID in Node.js instead of using MySQL UUID() function
    const fieldId = require('crypto').randomUUID();

    const fieldData: any = {
      id: fieldId,
      objectDefinitionId: obj.id,
      apiName: data.apiName,
      label: data.label,
      type: dbFieldType,
      description: data.description,
      isRequired: data.isRequired ?? false,
      isUnique: data.isUnique ?? false,
      referenceObject: referenceObject,
      defaultValue: data.defaultValue,
      length: data.length,
      precision: data.precision,
      scale: data.scale,
      created_at: knex.fn.now(),
      updated_at: knex.fn.now(),
    };
    
    // Merge UI metadata
    const uiMetadata: any = {};
    if (data.relationDisplayField) {
      uiMetadata.relationDisplayField = data.relationDisplayField;
    }
    if (data.uiMetadata) {
      Object.assign(uiMetadata, data.uiMetadata);
    }
    if (Object.keys(uiMetadata).length > 0) {
      fieldData.ui_metadata = JSON.stringify(uiMetadata);
    }
    
    await knex('field_definitions').insert(fieldData);
    const createdField = await knex('field_definitions').where({ id: fieldId }).first();

    // Add the column to the physical table
    const schemaManagementService = new SchemaManagementService();
    try {
      await schemaManagementService.addFieldToTable(knex, objectApiName, createdField);
      this.logger.log(`Added column ${data.apiName} to table for object ${objectApiName}`);
    } catch (error) {
      // If column creation fails, delete the field definition to maintain consistency
      this.logger.error(`Failed to add column ${data.apiName}: ${error.message}`);
      await knex('field_definitions').where({ id: fieldId }).delete();
      throw new Error(`Failed to create field column: ${error.message}`);
    }
    
    return createdField;
  }

  // Helper to get table name from object definition
  private getTableName(objectApiName: string): string {
    // Convert CamelCase to snake_case and pluralize
    // Account -> accounts, ContactPerson -> contact_persons
    const snakeCase = objectApiName
      .replace(/([A-Z])/g, '_$1')
      .toLowerCase()
      .replace(/^_/, '');
    
    // Simple pluralization (can be enhanced)
    if (snakeCase.endsWith('y')) {
      return snakeCase.slice(0, -1) + 'ies';
    } else if (snakeCase.endsWith('s')) {
      return snakeCase;
    } else {
      return snakeCase + 's';
    }
  }

  /**
   * Normalize field definition to ensure system fields are properly marked
   */
  private normalizeField(field: any): any {
    const systemFieldNames = ['id', 'tenantId', 'ownerId', 'created_at', 'updated_at', 'createdAt', 'updatedAt'];
    const isSystemField = systemFieldNames.includes(field.apiName);
    
    return {
      ...field,
      // Ensure system fields are marked correctly
      isSystem: isSystemField ? true : field.isSystem,
      isRequired: isSystemField ? false : field.isRequired,
      isCustom: isSystemField ? false : field.isCustom ?? true,
    };
  }

  /**
   * Convert frontend field type to database field type
   */
  private convertFrontendFieldType(frontendType: string): string {
    const typeMap: Record<string, string> = {
      'text': 'TEXT',
      'textarea': 'LONG_TEXT',
      'password': 'TEXT',
      'email': 'EMAIL',
      'number': 'NUMBER',
      'currency': 'CURRENCY',
      'percent': 'PERCENT',
      'select': 'PICKLIST',
      'multiSelect': 'MULTI_PICKLIST',
      'boolean': 'BOOLEAN',
      'date': 'DATE',
      'datetime': 'DATE_TIME',
      'time': 'TIME',
      'url': 'URL',
      'color': 'TEXT',
      'json': 'JSON',
      'lookup': 'LOOKUP',
      'belongsTo': 'LOOKUP',
      'hasMany': 'LOOKUP',
      'manyToMany': 'LOOKUP',
      'markdown': 'LONG_TEXT',
      'code': 'LONG_TEXT',
      'file': 'FILE',
      'image': 'IMAGE',
    };
    
    return typeMap[frontendType] || 'TEXT';
  }

  /**
   * Ensure a model is registered for the given object.
   * Delegates to ModelService which handles creating the model and all its dependencies.
   */
  private async ensureModelRegistered(
    tenantId: string,
    objectApiName: string,
  ): Promise<void> {
    // Provide a metadata fetcher function that the ModelService can use
    const fetchMetadata = async (apiName: string): Promise<ObjectMetadata> => {
      const objectDef = await this.getObjectDefinition(tenantId, apiName);
      const tableName = this.getTableName(apiName);

      // Build relations from lookup fields, but only for models that exist
      const lookupFields = objectDef.fields.filter((f: any) => 
        f.type === 'LOOKUP' && f.referenceObject
      );
      
      // Filter to only include relations where we can successfully resolve the target
      const validRelations: any[] = [];
      for (const field of lookupFields) {
        // Check if the referenced object will be available
        // We'll let the recursive registration attempt it, but won't include failed ones
        validRelations.push({
          name: field.apiName.replace(/Id$/, '').toLowerCase(),
          type: 'belongsTo' as const,
          targetObjectApiName: field.referenceObject,
          fromColumn: field.apiName,
          toColumn: 'id',
        });
      }

      return {
        apiName,
        tableName,
        fields: objectDef.fields.map((f: any) => ({
          apiName: f.apiName,
          label: f.label,
          type: f.type,
          isRequired: f.isRequired,
          isUnique: f.isUnique,
          referenceObject: f.referenceObject,
        })),
        relations: validRelations,
      };
    };

    // Let the ModelService handle recursive model creation
    try {
      await this.modelService.ensureModelWithDependencies(
        tenantId,
        objectApiName,
        fetchMetadata,
      );
    } catch (error) {
      this.logger.warn(
        `Failed to ensure model for ${objectApiName}: ${error.message}. Will fall back to manual hydration.`,
      );
    }
  }

  // Runtime endpoints - CRUD operations
  async getRecords(
    tenantId: string,
    objectApiName: string,
    userId: string,
    filters?: any,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Get user with roles and permissions
    const user = await User.query(knex)
      .findById(userId)
      .withGraphFetched('[roles.[objectPermissions, fieldPermissions]]');
    
    if (!user) {
      throw new NotFoundException('User not found');
    }
    
    // Get object definition with authorization settings
    const objectDefModel = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName })
      .withGraphFetched('fields');
    
    if (!objectDefModel) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }
    
    const tableName = this.getTableName(objectApiName);
    
    // Ensure model is registered
    await this.ensureModelRegistered(resolvedTenantId, objectApiName);

    // Use Objection model
    const boundModel = await this.modelService.getBoundModel(resolvedTenantId, objectApiName);
    let query = boundModel.query();
    
    // Apply authorization scope (modifies query in place)
    await this.authService.applyScopeToQuery(
      query,
      objectDefModel,
      user,
      'read',
      knex,
    );
    
    // Build graph expression for lookup fields
    const lookupFields = objectDefModel.fields?.filter(f => 
      f.type === 'LOOKUP' && f.referenceObject
    ) || [];
    
    if (lookupFields.length > 0) {
      // Build relation expression - use singular lowercase for relation name
      const relationExpression = lookupFields
        .map(f => f.apiName.replace(/Id$/, '').toLowerCase())
        .filter(Boolean)
        .join(', ');
      
      if (relationExpression) {
        query = query.withGraphFetched(`[${relationExpression}]`);
      }
    }
    
    // Apply additional filters
    if (filters) {
      query = query.where(filters);
    }
    
    const records = await query.select('*');
    
    // Filter fields based on field-level permissions
    const filteredRecords = await Promise.all(
      records.map(record => 
        this.authService.filterReadableFields(record, objectDefModel.fields, user)
      )
    );
    
    return filteredRecords;
  }

  async getRecord(
    tenantId: string,
    objectApiName: string,
    recordId: string,
    userId: string,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Get user with roles and permissions
    const user = await User.query(knex)
      .findById(userId)
      .withGraphFetched('[roles.[objectPermissions, fieldPermissions]]');
    
    if (!user) {
      throw new NotFoundException('User not found');
    }
    
    // Get object definition with authorization settings
    const objectDefModel = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName })
      .withGraphFetched('fields');
    
    if (!objectDefModel) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }
    
    // Ensure model is registered
    await this.ensureModelRegistered(resolvedTenantId, objectApiName);
    
    // Use Objection model
    const boundModel = await this.modelService.getBoundModel(resolvedTenantId, objectApiName);
    let query = boundModel.query().where({ id: recordId });
    
    // Apply authorization scope (modifies query in place)
    await this.authService.applyScopeToQuery(
      query,
      objectDefModel,
      user,
      'read',
      knex,
    );
    
    // Build graph expression for lookup fields
    const lookupFields = objectDefModel.fields?.filter(f => 
      f.type === 'LOOKUP' && f.referenceObject
    ) || [];
    
    if (lookupFields.length > 0) {
      // Build relation expression - use singular lowercase for relation name
      const relationExpression = lookupFields
        .map(f => f.apiName.replace(/Id$/, '').toLowerCase())
        .filter(Boolean)
        .join(', ');
      
      if (relationExpression) {
        query = query.withGraphFetched(`[${relationExpression}]`);
      }
    }
    
    const record = await query.first();
    if (!record) {
      throw new NotFoundException('Record not found');
    }
    
    // Filter fields based on field-level permissions
    const filteredRecord = await this.authService.filterReadableFields(record, objectDefModel.fields, user);
    
    return filteredRecord;
  }

  async createRecord(
    tenantId: string,
    objectApiName: string,
    data: any,
    userId: string,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Get user with roles and permissions
    const user = await User.query(knex)
      .findById(userId)
      .withGraphFetched('[roles.[objectPermissions, fieldPermissions]]');
    
    if (!user) {
      throw new NotFoundException('User not found');
    }
    
    // Get object definition with authorization settings
    const objectDefModel = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName })
      .withGraphFetched('fields');
    
    if (!objectDefModel) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }
    
    // Check if user has create permission
    const canCreate = await this.authService.canCreate(objectDefModel, user);
    if (!canCreate) {
      throw new NotFoundException('You do not have permission to create records of this object');
    }
    
    // Filter data to only editable fields
    const editableData = await this.authService.filterEditableFields(data, objectDefModel.fields, user);
    
    // Ensure model is registered
    await this.ensureModelRegistered(resolvedTenantId, objectApiName);
    
    // Use Objection model
    const boundModel = await this.modelService.getBoundModel(resolvedTenantId, objectApiName);
    const recordData = {
      ...editableData,
      ownerId: userId, // Auto-set owner
    };
    const record = await boundModel.query().insert(recordData);
    return record;
  }

  async updateRecord(
    tenantId: string,
    objectApiName: string,
    recordId: string,
    data: any,
    userId: string,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Get user with roles and permissions
    const user = await User.query(knex)
      .findById(userId)
      .withGraphFetched('[roles.[objectPermissions, fieldPermissions]]');
    
    if (!user) {
      throw new NotFoundException('User not found');
    }
    
    // Get object definition with authorization settings
    const objectDefModel = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName })
      .withGraphFetched('fields');
    
    if (!objectDefModel) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }
    
    const tableName = this.getTableName(objectApiName);
    
    // Get existing record
    const existingRecord = await knex(tableName).where({ id: recordId }).first();
    if (!existingRecord) {
      throw new NotFoundException('Record not found');
    }
    
    // Check if user can update this record
    await this.authService.assertCanPerformAction('update', objectDefModel, existingRecord, user, knex);
    
    // Filter data to only editable fields
    const editableData = await this.authService.filterEditableFields(data, objectDefModel.fields, user);
    
    // Remove system fields
    delete editableData.id;
    delete editableData.ownerId;
    delete editableData.created_at;
    delete editableData.tenantId;

    // Ensure model is registered
    await this.ensureModelRegistered(resolvedTenantId, objectApiName);

    // Use Objection model
    const boundModel = await this.modelService.getBoundModel(resolvedTenantId, objectApiName);
    await boundModel.query().where({ id: recordId }).update(editableData);
    return boundModel.query().where({ id: recordId }).first();
  }

  async deleteRecord(
    tenantId: string,
    objectApiName: string,
    recordId: string,
    userId: string,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Get user with roles and permissions
    const user = await User.query(knex)
      .findById(userId)
      .withGraphFetched('[roles.[objectPermissions, fieldPermissions]]');
    
    if (!user) {
      throw new NotFoundException('User not found');
    }
    
    // Get object definition with authorization settings
    const objectDefModel = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName });
    
    if (!objectDefModel) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }
    
    const tableName = this.getTableName(objectApiName);
    
    // Get existing record
    const existingRecord = await knex(tableName).where({ id: recordId }).first();
    if (!existingRecord) {
      throw new NotFoundException('Record not found');
    }
    
    // Check if user can delete this record
    await this.authService.assertCanPerformAction('delete', objectDefModel, existingRecord, user, knex);

    // Ensure model is registered
    await this.ensureModelRegistered(resolvedTenantId, objectApiName);

    // Use Objection model
    const boundModel = await this.modelService.getBoundModel(resolvedTenantId, objectApiName);
    await boundModel.query().where({ id: recordId }).delete();
    
    return { success: true };
  }

  /**
   * Update a field definition
   * Can update metadata (label, description, placeholder, helpText, etc.) safely
   * Cannot update apiName or type if field has existing data (prevent data loss)
   */
  async updateFieldDefinition(
    tenantId: string,
    objectApiName: string,
    fieldApiName: string,
    data: Partial<{
      label: string;
      description: string;
      isRequired: boolean;
      isUnique: boolean;
      defaultValue: string;
      placeholder: string;
      helpText: string;
      displayOrder: number;
      uiMetadata: Record<string, any>;
    }>,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);

    // Get the object definition
    const objectDef = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName });
    
    if (!objectDef) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }

    // Get the field definition
    const field = await knex('field_definitions')
      .where({ objectDefinitionId: objectDef.id, apiName: fieldApiName })
      .first();
    
    if (!field) {
      throw new NotFoundException(`Field ${fieldApiName} not found`);
    }

    // Check if this field has data (count records)
    const tableName = this.getTableName(objectApiName);
    const recordCount = await knex(tableName).count('* as cnt').first();
    const hasData = recordCount && (recordCount.cnt as number) > 0;

    // Prepare update object
    const updateData: any = {
      updated_at: knex.fn.now(),
    };

    // Always allow these updates
    if (data.label !== undefined) updateData.label = data.label;
    if (data.description !== undefined) updateData.description = data.description;
    if (data.displayOrder !== undefined) updateData.displayOrder = data.displayOrder;
    
    // Merge with existing uiMetadata
    const existingMetadata = field.ui_metadata ? JSON.parse(field.ui_metadata) : {};
    const newMetadata = { ...existingMetadata };
    
    if (data.placeholder !== undefined) newMetadata.placeholder = data.placeholder;
    if (data.helpText !== undefined) newMetadata.helpText = data.helpText;
    if (data.uiMetadata) {
      Object.assign(newMetadata, data.uiMetadata);
    }

    if (Object.keys(newMetadata).length > 0) {
      updateData.ui_metadata = JSON.stringify(newMetadata);
    }

    // Conditional updates based on data existence
    if (data.isRequired !== undefined) {
      if (hasData && data.isRequired && !field.isRequired) {
        throw new Error('Cannot make a field required when data exists. Existing records may have null values.');
      }
      updateData.isRequired = data.isRequired;
    }

    if (data.isUnique !== undefined) {
      if (hasData && data.isUnique && !field.isUnique) {
        throw new Error('Cannot add unique constraint to field with existing data. Existing records may have duplicate values.');
      }
      updateData.isUnique = data.isUnique;
    }

    // Update the field definition
    await knex('field_definitions')
      .where({ id: field.id })
      .update(updateData);

    return knex('field_definitions').where({ id: field.id }).first();
  }

  /**
   * Delete a field definition and clean up dependencies
   * Removes the column from the physical table
   * Removes field references from page layouts
   * CASCADE deletion handles role_field_permissions
   */
  async deleteFieldDefinition(
    tenantId: string,
    objectApiName: string,
    fieldApiName: string,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);

    // Get the object definition
    const objectDef = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName });
    
    if (!objectDef) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }

    // Get the field definition
    const field = await knex('field_definitions')
      .where({ objectDefinitionId: objectDef.id, apiName: fieldApiName })
      .first();
    
    if (!field) {
      throw new NotFoundException(`Field ${fieldApiName} not found`);
    }

    // Prevent deletion of system fields
    const systemFieldNames = ['id', 'tenantId', 'ownerId', 'created_at', 'updated_at', 'createdAt', 'updatedAt'];
    if (systemFieldNames.includes(fieldApiName)) {
      throw new Error(`Cannot delete system field: ${fieldApiName}`);
    }

    // Clean up page layouts - remove field references from layoutConfig
    try {
      const pageLayouts = await knex('page_layouts')
        .where({ object_id: objectDef.id });
      
      for (const layout of pageLayouts) {
        // Handle JSON column that might already be parsed
        let layoutConfig;
        if (layout.layout_config) {
          layoutConfig = typeof layout.layout_config === 'string' 
            ? JSON.parse(layout.layout_config) 
            : layout.layout_config;
        } else {
          layoutConfig = { fields: [] };
        }
        
        // Filter out any field references for this field
        if (layoutConfig.fields) {
          layoutConfig.fields = layoutConfig.fields.filter(
            (f: any) => f.fieldId !== field.id,
          );
        }
        
        // Update the page layout
        await knex('page_layouts')
          .where({ id: layout.id })
          .update({
            layout_config: JSON.stringify(layoutConfig),
            updated_at: knex.fn.now(),
          });
      }
    } catch (error) {
      // If page layouts table doesn't exist or query fails, log but continue
      this.logger.warn(`Could not update page layouts for field deletion: ${error.message}`);
    }

    // Clean up dependsOn references in other fields
    const otherFields = await knex('field_definitions')
      .where({ objectDefinitionId: objectDef.id })
      .whereNot({ id: field.id });
    
    for (const otherField of otherFields) {
      // Handle JSON column that might already be parsed
      let metadata;
      if (otherField.ui_metadata) {
        metadata = typeof otherField.ui_metadata === 'string'
          ? JSON.parse(otherField.ui_metadata)
          : otherField.ui_metadata;
      } else {
        metadata = {};
      }
      
      if (metadata.dependsOn && Array.isArray(metadata.dependsOn)) {
        metadata.dependsOn = metadata.dependsOn.filter(
          (dep: any) => dep !== field.apiName,
        );
        
        await knex('field_definitions')
          .where({ id: otherField.id })
          .update({
            ui_metadata: JSON.stringify(metadata),
            updated_at: knex.fn.now(),
          });
      }
    }

    // Remove the column from the physical table
    const tableName = this.getTableName(objectApiName);
    const schemaManagementService = new SchemaManagementService();
    
    try {
      await schemaManagementService.removeFieldFromTable(knex, objectApiName, fieldApiName);
    } catch (error) {
      this.logger.warn(`Failed to remove column ${fieldApiName} from table ${tableName}: ${error.message}`);
      // Continue with deletion even if column removal fails - field definition must be cleaned up
    }

    // Delete the field definition (CASCADE will delete role_field_permissions)
    await knex('field_definitions').where({ id: field.id }).delete();

    return { success: true };
  }

  async getFieldPermissions(tenantId: string, objectId: string) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Get all field permissions for this object's fields
    const permissions = await knex('role_field_permissions as rfp')
      .join('field_definitions as fd', 'fd.id', 'rfp.fieldDefinitionId')
      .where('fd.objectDefinitionId', objectId)
      .select('rfp.*');
    
    return permissions;
  }

  async updateFieldPermission(
    tenantId: string,
    roleId: string,
    fieldDefinitionId: string,
    canRead: boolean,
    canEdit: boolean,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Check if permission already exists
    const existing = await knex('role_field_permissions')
      .where({ roleId, fieldDefinitionId })
      .first();
    
    if (existing) {
      // Update existing permission
      await knex('role_field_permissions')
        .where({ roleId, fieldDefinitionId })
        .update({
          canRead,
          canEdit,
          updated_at: knex.fn.now(),
        });
    } else {
      // Create new permission
      await knex('role_field_permissions').insert({
        id: knex.raw('(UUID())'),
        roleId,
        fieldDefinitionId,
        canRead,
        canEdit,
        created_at: knex.fn.now(),
        updated_at: knex.fn.now(),
      });
    }
    
    return { success: true };
  }

  async getObjectPermissions(
    tenantId: string,
    objectApiName: string,
    roleId: string,
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Get object definition
    const objectDef = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName });
    
    if (!objectDef) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }
    
    // Get role object permissions
    const permission = await knex('role_object_permissions')
      .where({ roleId, objectDefinitionId: objectDef.id })
      .first();
    
    if (!permission) {
      // Return default permissions (all false)
      return {
        canCreate: false,
        canRead: false,
        canEdit: false,
        canDelete: false,
        canViewAll: false,
        canModifyAll: false,
      };
    }
    
    return {
      canCreate: Boolean(permission.canCreate),
      canRead: Boolean(permission.canRead),
      canEdit: Boolean(permission.canEdit),
      canDelete: Boolean(permission.canDelete),
      canViewAll: Boolean(permission.canViewAll),
      canModifyAll: Boolean(permission.canModifyAll),
    };
  }

  async updateObjectPermissions(
    tenantId: string,
    objectApiName: string,
    data: {
      roleId: string;
      canCreate: boolean;
      canRead: boolean;
      canEdit: boolean;
      canDelete: boolean;
      canViewAll: boolean;
      canModifyAll: boolean;
    },
  ) {
    const resolvedTenantId = await this.tenantDbService.resolveTenantId(tenantId);
    const knex = await this.tenantDbService.getTenantKnexById(resolvedTenantId);
    
    // Get object definition
    const objectDef = await ObjectDefinition.query(knex)
      .findOne({ apiName: objectApiName });
    
    if (!objectDef) {
      throw new NotFoundException(`Object ${objectApiName} not found`);
    }
    
    // Check if permission already exists
    const existing = await knex('role_object_permissions')
      .where({ roleId: data.roleId, objectDefinitionId: objectDef.id })
      .first();
    
    if (existing) {
      // Update existing permission
      await knex('role_object_permissions')
        .where({ roleId: data.roleId, objectDefinitionId: objectDef.id })
        .update({
          canCreate: data.canCreate,
          canRead: data.canRead,
          canEdit: data.canEdit,
          canDelete: data.canDelete,
          canViewAll: data.canViewAll,
          canModifyAll: data.canModifyAll,
          updated_at: knex.fn.now(),
        });
    } else {
      // Create new permission
      await knex('role_object_permissions').insert({
        id: knex.raw('(UUID())'),
        roleId: data.roleId,
        objectDefinitionId: objectDef.id,
        canCreate: data.canCreate,
        canRead: data.canRead,
        canEdit: data.canEdit,
        canDelete: data.canDelete,
        canViewAll: data.canViewAll,
        canModifyAll: data.canModifyAll,
        created_at: knex.fn.now(),
        updated_at: knex.fn.now(),
      });
    }
    
    return { success: true };
  }
}