Create Spec

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/dpf-it-com/json-schema/dpf-it-com-create-spec-request-schema.json",
  "title": "Create Spec",
  "description": "`sourceType` defaults to `file`. When `file`, `sampleFileName` is required (as\nbelow). When `tables`, `sampleFileName`/`formatFileName` do not apply; instead\n`sourceTables` (the source table(s) to query) is required. The target is a\ngenerated SQL `INSERT`/`MERGE` (see `additionalPrompt`), executed via the query\nengine instead of a Glue job. See `start-analysis`.\n\nWhen `compaction`, the spec bin-packs the small data files of the tables in\n`sourceTables` and produces no new data at all. It therefore takes **no target of\nany kind** — `targetOption`, `targetTables` and `targetSchemaFileName` are\nrejected, as are `sampleFileName`/`formatFileName` — and there is nothing to\nanalyse, so **do not call `start-analysis`**: the spec is created with status\n`ready` and is run with `create-job` + `start-job`, or on a `schedule` /\n`spec_success` trigger. See `expirePriorSnapshots` for whether a run also reclaims\nstorage. Billed like any other data job at the spec's `computeSize`.\n\nAll three `targetOption` values work for both source types, and the target table\nrules are the same either way: for `sourceType: tables`, `existing-tables` fetches\nthe named table's schema (requires `targetTables`, exactly one entry); `target-schema-file`\ncreates it from an uploaded schema file; `auto-infer` has the AI design its schema\n(and the query producing it) together. For the latter two, `targetTables` is\noptional — omit it to let the target table's name be derived automatically (from\nthe schema file, or AI-chosen), or pass exactly one entry to pin the name yourself.\n",
  "x-generated": "2026-11-06",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/dpf-it-com-openapi.yml#/components/schemas/CreateSpecRequest",
  "type": "object",
  "required": [
    "action",
    "workspaceId",
    "specName"
  ],
  "properties": {
    "action": {
      "type": "string",
      "enum": [
        "create-spec"
      ],
      "description": "Action type - must be \"create-spec\""
    },
    "workspaceId": {
      "type": "string",
      "format": "uuid",
      "description": "Workspace ID that owns this spec"
    },
    "specName": {
      "type": "string",
      "description": "Unique name for this data spec within the workspace",
      "minLength": 1,
      "maxLength": 255
    },
    "sourceType": {
      "type": "string",
      "enum": [
        "file",
        "tables",
        "compaction"
      ],
      "default": "file",
      "description": "Where the data comes from.\n- **file**: upload a sample data file (default) — see `sampleFileName`.\n- **tables**: query existing Iceberg table(s) — see `sourceTables`. Runs via\n  the query engine, not a Glue job.\n- **compaction**: rewrite the small data files of existing Iceberg table(s) —\n  see `sourceTables` and `expirePriorSnapshots`. Moves no data, writes no new\n  table, and takes no target. `targetOption` is required for the other two\n  source types and rejected for this one.\n"
    },
    "sourceTables": {
      "type": "array",
      "description": "Required if sourceType=tables (the source table(s) the generated query reads\nfrom) or sourceType=compaction (the table(s) to compact).\n",
      "items": {
        "type": "string"
      }
    },
    "expirePriorSnapshots": {
      "type": "boolean",
      "default": false,
      "description": "sourceType=compaction only. Controls whether a run also reclaims storage.\n\n- **false** (default): the compacted files are committed as a new snapshot and\n  nothing else changes. Prior snapshots still reference the files that were\n  replaced, so those files remain and rollback across the compaction boundary\n  still works — but no storage is freed.\n- **true**: in addition, every snapshot older than the compaction commit is\n  expired and every replaced file that no remaining snapshot references is\n  deleted, **in the same run**. This frees the storage immediately and reports\n  the bytes reclaimed on the job, at the cost of being unable to roll back to\n  before the compaction.\n"
    },
    "sampleFileName": {
      "type": "string",
      "description": "Required if sourceType=file (default). Name of the sample data file to upload\n(e.g., data.csv, data.dat, data.parquet). This is the actual data that will be\nparsed and loaded into target tables. **Must be 100 KB or smaller** (representative\nsample only).\n",
      "minLength": 1,
      "maxLength": 255
    },
    "formatFileName": {
      "type": [
        "string",
        "null"
      ],
      "description": "Optional, sourceType=file only: Name of the format specification file to upload\n(e.g., schema.json, format.xls). This file describes how to parse the sample data\n(field positions, record types, etc.). If not provided, the AI will infer the\nformat from the sample file. **Must be 100 KB or smaller.**\n",
      "minLength": 1,
      "maxLength": 255
    },
    "targetOption": {
      "type": "string",
      "enum": [
        "existing-tables",
        "target-schema-file",
        "auto-infer"
      ],
      "description": "**Required for sourceType=file and sourceType=tables; rejected for\nsourceType=compaction**, which writes to no target at all.\n\nHow to determine the target schema:\n- **existing-tables**: Use table(s) already in the Iceberg catalog (requires targetTables).\n- **target-schema-file**: Provide a target schema file (requires targetSchemaFileName);\n  the table is created from it.\n- **auto-infer**: AI infers the schema — from the sample + format files for\n  sourceType=file, or from the source table(s) + additionalPrompt for\n  sourceType=tables — and creates the table.\n"
    },
    "targetTables": {
      "type": "array",
      "description": "Required if targetOption=existing-tables (array of target table names — sourceType=tables\nallows exactly one entry, the query's single target; sourceType=file allows one or more).\nOptional otherwise: for targetOption=target-schema-file/auto-infer the target table (and\nits name) is derived automatically — from the schema file, or AI-designed — for both\nsourceType values. Pass exactly one entry (sourceType=tables) to pin the name yourself\ninstead of letting it be derived.\n",
      "items": {
        "type": "string"
      }
    },
    "targetSchemaFileName": {
      "type": "string",
      "description": "Required if targetOption=target-schema-file. Name of the target schema file to upload.\nSent to AI to derive the target tables and field mapping. **Must be 100 KB or smaller.**\n",
      "minLength": 1,
      "maxLength": 255
    },
    "description": {
      "type": "string",
      "description": "Optional description of the data load spec",
      "maxLength": 1000
    },
    "merge": {
      "type": "boolean",
      "description": "Optional: Enable merge/upsert mode using primary keys instead of append.\nWhen sourceType=tables, also controls the generated SQL shape: MERGE when\ntrue, INSERT when false/omitted. When sourceType=file with targetOption\nexisting-tables, the target table must already have a primary key set\n(400 MISSING_PRIMARY_KEY otherwise) — set one via the Iceberg REST endpoint\n(POST /iceberg/v1/namespaces/{namespace}/tables/{table}, set-properties on\ndpf.primary-keys). New tables (targetOption auto-infer/target-schema-file)\nhave the AI choose one automatically when merge is true.\n"
    },
    "computeSize": {
      "type": "string",
      "enum": [
        "small",
        "large"
      ],
      "description": "Controls processing power for the job — the data load Glue job (sourceType=file)\nor credit-charging tier for the generated query (sourceType=tables).\n- **small**: 1/16 DPU (default, suitable for most workloads)\n- **large**: 1 DPU (for larger or more complex data transformations)\n",
      "default": "small"
    },
    "additionalPrompt": {
      "type": "string",
      "description": "Optional: Additional instructions for the AI. For sourceType=file: parsing,\nschema inference, or loading logic. For sourceType=tables: what the generated\nSQL query should compute from the source table(s).\n",
      "maxLength": 2000
    }
  }
}