{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://runninglake.com/schema/v1alpha1/source.json", "title": "Source", "description": "A Source as RunningLake's configuration repository declares it, at sources/.yaml. Generated from the Go types the control plane decodes this file into, so it checks the shape the gate reads — field names and types, and which fields may exist. The gate also applies rules no schema can express — required fields inside spec among them — so a file this schema accepts can still be refused by `rl diff`. Take #/properties/spec for the spec alone.", "type": "object", "additionalProperties": false, "required": [ "apiVersion", "kind", "metadata" ], "properties": { "apiVersion": { "type": "string", "const": "runninglake.io/v1alpha1", "description": "The config schema's version, which is independent of the HTTP API's (rule unsupported-api-version)." }, "kind": { "type": "string", "const": "Source", "description": "The kind this file declares. A kind that disagrees with the file's path is refused (rule kind-mismatch)." }, "metadata": { "required": [ "name" ], "type": "object", "additionalProperties": false, "properties": { "name": { "type": "string", "pattern": "^[a-z0-9_-]{1,63}$", "description": "The resource's name, which is also its path segment. It must be the name this file's path gives it (rule name-mismatch)." }, "labels": { "type": "object", "additionalProperties": { "type": "string" } }, "annotations": { "type": "object", "additionalProperties": { "type": "string" } } } }, "spec": { "description": "SourceSpec says what a Source reads and where it lands. Nothing runs a sync yet: this declares the shape a connector will read once ingestion exists.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "type": { "description": "Type is the connection's kind: postgres_cdc, mysql_cdc, jdbc, api, object_storage or stream. It decides which other fields this Source may and must set.", "type": [ "string", "null" ] }, "host": { "description": "Host is the database's address. Required for postgres_cdc, mysql_cdc and jdbc; not used by the other types, which connect a different way (connector, url or brokers).", "type": [ "string", "null" ] }, "port": { "description": "Port is the database's port. Required alongside Host.", "type": [ "integer", "null" ], "minimum": -9223372036854775808, "maximum": 9223372036854775807 }, "database": { "description": "Database is the database's name. Required alongside Host.", "type": [ "string", "null" ] }, "engine": { "description": "Engine is the database a jdbc Source reads: postgres or mysql. SQL Server and Oracle are not offered yet; they join with the connector runtime. Required for jdbc, which reads tables by SQL the engine decides the dialect of; refused for every other type (change capture names its database by its type, and the others do not connect to one). It is checked and recorded, and nothing reads a Source yet.", "type": [ "string", "null" ] }, "credentials": { "description": "Credentials says where this Source's credentials live, by reference. A Source has no field for a secret value, and none belongs in git.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "secretRef": { "description": "SecretRef is the name of a SecretReference in this project, which says where the credentials live in your own store.", "type": [ "string", "null" ] }, "user": { "description": "User is the database user the connector is meant to sign in as, for a Source that connects to a database (postgres_cdc, mysql_cdc, jdbc). A user name is a role and not a secret, so it is written here beside the reference and is held to the shape of an identifier, never to the entropy test a password is: the setup SQL you run on the database names this role.", "type": [ "string", "null" ] } } }, "read": { "description": "Read names what this Source reads: the one list matching Type (tables for postgres_cdc, mysql_cdc and jdbc; endpoints for api; objects for object_storage; topics for stream). A name found by a wildcard would be discovered, not declared, so a pattern such as public.* is refused: every name here is one a person typed.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "tables": { "description": "Tables is the tables a postgres_cdc, mysql_cdc or jdbc Source reads, as the source database names them: public.orders. Each lands in a Table of the schema in land.schema called by the last part of its name in lower case, so public.orders lands in bronze.crm.orders; declare that Table in the same change, with no schema of its own.", "type": [ "array", "null" ], "items": { "type": [ "string", "null" ] } }, "endpoints": { "description": "Endpoints is the endpoints an api Source reads, as its connector names them.", "type": [ "array", "null" ], "items": { "type": [ "string", "null" ] } }, "objects": { "description": "Objects is the object key prefixes an object_storage Source is meant to copy. Each is a folder written relative to the Source's url, such as daily/, so url s3://partner-storage/exports/ and folder daily/ name s3://partner-storage/exports/daily (the folder, written here without its last slash). It starts with no /, holds no whole address (://), no . or .. folder name and no empty folder name, and daily and daily/ are the same folder.", "type": [ "array", "null" ], "items": { "type": [ "string", "null" ] } }, "topics": { "description": "Topics is the topics a stream Source reads.", "type": [ "array", "null" ], "items": { "type": [ "string", "null" ] } } } }, "land": { "description": "Land names where this Source's rows are meant to arrive: Schema, a declared Schema (meant to be in the bronze layer; the gate does not check it), for every type but object_storage, which names Volume instead. One declared Table is created per Read item, named from it, and a table has one Source.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "schema": { "description": "Schema is a declared Schema, such as bronze.crm, meant to be in the bronze layer; the gate does not check it. Every type but object_storage sets this.", "type": [ "string", "null" ] }, "volume": { "description": "Volume is a declared Volume. Only object_storage sets this.", "type": [ "string", "null" ] } } }, "apply": { "description": "Apply names a declared silver Schema this Source's changes are meant to be applied to. It is checked and recorded, and nothing applies changes to it yet. Optional: a Source may land in bronze alone. Each table in it declares its own key as its Iceberg schema's identifier fields.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "schema": { "description": "Schema is a declared Schema in the silver layer, such as silver.crm.", "type": [ "string", "null" ] } } }, "schedule": { "description": "Schedule is when a batch Source is meant to sync by itself, reusing a pipeline's schedule mapping. It is checked and recorded, and nothing syncs a Source on it yet. A continuous Source (postgres_cdc, mysql_cdc, stream) is meant to run all the time and takes no schedule.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "cron": { "description": "Cron is when it runs: five fields (minute, hour, day of month, month, day of week), or @hourly, @daily, @weekly, @monthly or @yearly. A seconds field and Quartz's L, W, # and ? are refused.", "type": [ "string", "null" ] }, "timezone": { "description": "Timezone is the IANA zone the cron is read in, such as Europe/London. Default UTC. Abbreviations such as EST, and Local, are refused: they name no place's rules.", "type": [ "string", "null" ] }, "paused": { "description": "Paused stops the schedule from starting runs. Resuming does not replay the paused period. A run started by hand still runs.", "type": [ "boolean", "null" ] }, "catchup": { "description": "Catchup is what happens to runs missed while the platform could not start them: latest (default) runs the newest one, all runs the newest 5, none runs none.", "type": [ "string", "null" ] }, "overlap": { "description": "Overlap is what a scheduled start does while a run is still active: skip (default) records it skipped, queue waits up to maxQueuedRuns, cancel_previous cancels the active run.", "type": [ "string", "null" ] }, "maxConcurrentRuns": { "description": "MaxConcurrentRuns is how many runs may be active at once. Default 1, and only a partitioned pipeline may say more: two runs of one table would write it twice.", "type": [ "integer", "null" ], "minimum": -9223372036854775808, "maximum": 9223372036854775807 }, "maxQueuedRuns": { "description": "MaxQueuedRuns is how many runs overlap: queue may hold waiting. Default 20.", "type": [ "integer", "null" ], "minimum": -9223372036854775808, "maximum": 9223372036854775807 }, "jitter": { "description": "Jitter delays each scheduled start by up to this long, such as 30s or 2m, at most 5m; the same delay every time for one pipeline. Default none.", "type": [ "string", "null" ] } } }, "mode": { "description": "Mode is continuous (postgres_cdc, mysql_cdc, stream) or batch (jdbc, api, object_storage): it must agree with Type, and says it explicitly so a diff shows it.", "type": [ "string", "null" ] }, "incremental": { "description": "Incremental is meant to read only the rows newer than the last sync, by Cursor, a column such as updated_at. It is checked and recorded, and nothing reads a Source yet. Only jdbc may set it; every other type is meant to read its declared items whole or by their own protocol.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "cursor": { "description": "Cursor is the column an incremental sync orders by, such as updated_at. Required whenever Incremental is set: a cursor-less incremental sync has nothing to read from.", "type": [ "string", "null" ] }, "lookback": { "description": "Lookback re-reads this far behind the cursor's own maximum, such as 10m, to catch rows committed out of order. Leave it out for none.", "type": [ "string", "null" ] } } }, "drift": { "description": "Drift is what a Source is meant to do when its own schema changes underneath it. It is checked and recorded, and nothing detects a schema change yet. Leave it out for the default, additive.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "policy": { "description": "Policy is additive (the default: added or widened columns are meant to apply; anything else is meant to be rescued, never blocking the sync), propose_all (additive changes are meant to be rescued too, until approved) or pause (the Source is meant to stop at the first change). It is checked and recorded, and nothing acts on it yet.", "type": [ "string", "null" ] } } }, "network": { "description": "Network is meant to name the addresses this Source's pods may reach INSTEAD of whatever Host resolves to, for when resolving Host at connection time is not to be trusted. It is checked and recorded, and nothing enforces it yet.", "type": [ "object", "null" ], "additionalProperties": false, "properties": { "allow": { "description": "Allow lists the addresses to trust in place of Host's own resolution, each a single host: a.b.c.d/32 or an IPv6 /128. Host is meant not to be looked up while it is set, so list every address the Source connects to. A broader range is refused, because trusting a /16 for one database would, once it is enforced, let the pod reach every other address in it too.", "type": [ "array", "null" ], "items": { "type": [ "string", "null" ] } } } }, "connector": { "description": "Connector is meant to be the catalogue id of an api Source's connector, such as stripe; no catalogue exists yet. Only api may set it.", "type": [ "string", "null" ] }, "url": { "description": "URL is an object_storage Source's own address, such as s3://partner-storage/exports/ or sftp://drop.partner.example.com/outbound/. The schemes a plane is meant to copy from are s3, gs, abfs, abfss and sftp, in lower case; another scheme is accepted with a warning, and nothing here checks that it can be read. The address carries no password: the credential is named by credentials.secretRef. Every folder in read.objects is written relative to this address. Only object_storage may set it.", "type": [ "string", "null" ] }, "hostKeyFingerprint": { "description": "HostKeyFingerprint pins the SSH host key of an sftp address, written as ssh-keygen -l prints it: the word SHA256, a colon, and the 43 characters of unpadded base64 that follow it. Asked for every sftp address: a Source declared without one is warned about, and a change that declares one without it is refused. Any other address refuses it. It is meant to be the only host key the plane accepts: a server whose key differs is meant to be refused, and the plane is meant never to trust the first key it sees. An MD5 fingerprint is not accepted. Nothing connects to the server yet. Only object_storage may set it.", "type": [ "string", "null" ] }, "filePattern": { "description": "FilePattern says which files under each folder of read.objects are meant to be copied, written relative to that folder and matched against the whole of a file's path below it, case-sensitively. Only * (any run of characters except /) and ? (one character except /) are special, so */*.csv is the csv files one folder down and *.csv is the csv files directly in the folder; a pattern is written one * for each folder level, and ** is not accepted. A pattern is at most 1024 bytes, starts with no / and does not end with /, and holds no [ ], { } or backslash, no . or .. or empty folder name, no invalid text, no space at either end and no zero-width, text-direction, line-separator or control character. Leave it out and no filter is meant to apply to the files under each folder. Nothing copies a file yet. Only object_storage may set it.", "type": [ "string", "null" ] }, "brokers": { "description": "Brokers lists a stream Source's broker addresses. Only stream may set it, alongside Format.", "type": [ "array", "null" ], "items": { "type": [ "string", "null" ] } }, "format": { "description": "Format is a stream Source's message encoding, such as avro or json. Only stream may set it, alongside Brokers.", "type": [ "string", "null" ] }, "paused": { "description": "Paused says this Source is meant to be stopped, and is changed through a change set, the way a pipeline's own Paused is. The control plane records it as the state paused, and nothing is stopped by it yet, because nothing reads a Source.", "type": [ "boolean", "null" ] } } } } }