{ "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ArkFlow engine configuration", "type": "object", "additionalProperties": false, "properties": { "logging": { "type": "object", "description": "Logging configuration.", "additionalProperties": false, "properties": { "level": { "type": "string", "enum": [ "trace", "debug", "info", "warn", "error" ], "default": "info", "description": "Log level." }, "file_path": { "type": "string", "description": "Optional path to a log file." }, "format": { "type": "string", "enum": [ "plain", "json" ], "default": "plain", "description": "Log output format." } } }, "health_check": { "type": "object", "description": "Health check HTTP server, control-plane API, agent registration, data-plane, and process observability configuration.", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": true }, "address": { "type": "string", "default": "127.0.0.1:8080" }, "health_path": { "type": "string", "default": "/health" }, "readiness_path": { "type": "string", "default": "/readiness" }, "liveness_path": { "type": "string", "default": "/liveness" }, "api_prefix": { "type": "string", "default": "/api/v1" }, "api_token": { "type": "string", "writeOnly": true }, "insecure_local": { "type": "boolean", "default": false, "description": "Explicit opt-in to run the standalone control API without a token on a non-loopback bind; the process logs a prominent warning." }, "cors_origins": { "type": "array", "items": { "type": "string" }, "default": [] }, "hub_urls": { "type": "array", "items": { "type": "string" }, "default": [], "description": "Hub addresses for compute-node Agent mode, tried in order as failover candidates. Empty keeps standalone mode." }, "hub_url": { "type": "string", "deprecated": true, "description": "Removed single-address key; any occurrence fails with a migration hint. Use hub_urls." }, "node_id": { "type": "string", "description": "Stable identity used when this process reports to a Hub." }, "node_token": { "type": "string", "writeOnly": true, "description": "Shared node registration credential. Never included in reports." }, "agent_lease_ttl_ms": { "type": "integer", "format": "uint64", "default": 15000, "description": "Lease duration advertised by a compute node to its Hub." }, "agent_session_ttl_ms": { "type": "integer", "format": "uint64", "default": 3600000, "description": "Lifetime of a Hub-issued agent session credential." }, "data_port": { "type": "integer", "format": "uint16", "description": "Data-plane listen port for cross-node shuffle. Absent keeps the node colocated-only." }, "data_host": { "type": "string", "description": "Routable host advertised to peers for the data plane (for example the node's LAN IP)." }, "observability": { "type": "object", "description": "Process-level observability export (metrics, readiness, liveness); available even when the control-plane API server is disabled.", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": true }, "address": { "type": "string", "default": "127.0.0.1:8081" }, "metrics_path": { "type": "string", "default": "/metrics" }, "ready_path": { "type": "string", "default": "/ready" }, "live_path": { "type": "string", "default": "/live" }, "tracing": { "type": "object", "description": "OTel trace export (disabled by default).", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": false }, "endpoint": { "type": "string", "default": "http://127.0.0.1:4318/v1/traces" }, "service_name": { "type": "string", "default": "arkflow" } } } } } } }, "streams": { "type": "array", "description": "List of stream processing pipelines.", "items": { "$ref": "#/$defs/stream" } }, "jobs": { "type": "array", "description": "Local Jobs declared directly in the configuration; they execute on the unified kernel without a Hub.", "items": { "$ref": "#/$defs/job" } } }, "anyOf": [ { "required": [ "streams" ] }, { "required": [ "jobs" ] } ], "$defs": { "input": { "type": "object", "description": "Select a registered input component.", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ null, null, null, null, null, null, null, null, null, null, null, null, null ], "description": "input component type." }, "name": { "type": "string", "description": "Optional logical name for this component instance." } }, "oneOf": [ { "type": "object", "title": "file", "description": "Reads records from local or remote object storage (S3, GCS, Azure, HDFS) in CSV/JSON/Parquet/Avro/Arrow formats.", "properties": { "type": { "type": "string", "const": "file", "description": "Reads records from local or remote object storage (S3, GCS, Azure, HDFS) in CSV/JSON/Parquet/Avro/Arrow formats." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "input_type": { "type": "object", "description": "Format-specific input settings (type: csv/json/parquet/avro/arrow)." }, "ballista": { "type": "object", "description": "Optional Ballista distributed compute configuration." }, "query": { "type": "object", "description": "Optional SQL query and table name to filter the read." } }, "required": [ "type" ], "examples": [ { "input_type": { "type": "parquet", "path": "s3://bucket/data.parquet" } } ] }, { "type": "object", "title": "generate", "description": "Generates synthetic text messages on a fixed interval (useful for testing and load simulation).", "properties": { "type": { "type": "string", "const": "generate", "description": "Generates synthetic text messages on a fixed interval (useful for testing and load simulation)." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "context": { "type": "string", "description": "Payload string emitted on every read." }, "interval": { "type": "string", "description": "Delay between batches (e.g. '100ms', '1s'). Accepts any humantime duration." }, "count": { "type": "integer", "minimum": 1, "description": "Optional total number of messages to emit before signalling EOF." }, "batch_size": { "type": "integer", "minimum": 1, "default": 1, "description": "Number of messages returned per read call." } }, "required": [ "type" ], "examples": [ { "context": "hello", "interval": "1s", "count": 100, "batch_size": 10 } ] }, { "type": "object", "title": "http", "description": "Receives data via HTTP. Can run as a server (POST/PUT on `path`) or poll a remote endpoint.", "properties": { "type": { "type": "string", "const": "http", "description": "Receives data via HTTP. Can run as a server (POST/PUT on `path`) or poll a remote endpoint." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "address": { "type": "string", "description": "Bind address (server mode) or base URL (client mode)." }, "path": { "type": "string", "description": "Path to accept messages on (server mode) or full request path (client mode)." }, "method": { "type": "string", "enum": [ "GET", "POST", "PUT", "DELETE", "PATCH" ], "description": "HTTP method (client mode)." }, "interval": { "type": "string", "description": "Polling interval in humantime format (client mode, e.g. '5s')." }, "cors_enabled": { "type": "boolean", "default": false, "description": "Enable CORS for the server." }, "auth": { "type": "object", "description": "Authentication configuration." } }, "required": [ "type" ], "examples": [ { "address": "0.0.0.0:8080", "path": "/ingest", "cors_enabled": true } ] }, { "type": "object", "title": "kafka", "description": "Consumes messages from Apache Kafka topics with a consumer group.", "properties": { "type": { "type": "string", "const": "kafka", "description": "Consumes messages from Apache Kafka topics with a consumer group." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "brokers": { "type": "array", "items": { "type": "string" }, "description": "List of Kafka broker addresses." }, "topics": { "type": "array", "items": { "type": "string" }, "description": "Topics to subscribe to." }, "consumer_group": { "type": "string", "description": "Consumer group ID for offset coordination." }, "client_id": { "type": "string", "description": "Optional client identifier." }, "start_from_latest": { "type": "boolean", "default": false, "description": "When true, ignore committed offsets and start from the latest message." }, "fetch_min_bytes": { "type": "integer", "minimum": 0, "description": "Minimum bytes before the broker responds to a fetch request." }, "fetch_max_bytes": { "type": "integer", "minimum": 0, "description": "Maximum bytes for a fetch request." }, "fetch_max_partition_bytes": { "type": "integer", "minimum": 0, "description": "Maximum bytes per partition in a fetch request." }, "fetch_wait_max_ms": { "type": "integer", "minimum": 0, "description": "Maximum time to wait for fetch data in milliseconds." }, "batch_max_rows": { "type": "integer", "minimum": 1, "default": 1024, "description": "Maximum number of messages aggregated into one read batch (clamped to 1 minimum; 1 restores per-message batches)." }, "batch_max_bytes": { "type": "integer", "minimum": 1, "default": 8388608, "description": "Maximum accumulated payload bytes per read batch; the first message is always included (clamped to 1 minimum)." }, "security": { "type": "object", "description": "SASL authentication and TLS settings. Omit entirely for plaintext.", "properties": { "protocol": { "type": "string", "enum": [ "plaintext", "ssl", "sasl_plaintext", "sasl_ssl" ], "description": "Security protocol. Defaults to inferred from which sub-blocks are present: both -> sasl_ssl, only sasl -> sasl_plaintext, only tls -> ssl, neither -> plaintext." }, "sasl": { "type": "object", "description": "SASL credentials (PLAIN or SCRAM).", "properties": { "mechanism": { "type": "string", "enum": [ "plain", "scram-sha-256", "scram-sha-512" ], "description": "SASL mechanism." }, "username": { "type": "string", "description": "SASL username (required for plain/scram)." }, "password": { "type": "string", "description": "SASL password (required for plain/scram)." } }, "required": [ "mechanism" ] }, "tls": { "type": "object", "description": "TLS settings. ca/cert/key accept a file path or inline PEM text (auto-detected).", "properties": { "ca": { "type": "string", "description": "CA certificate for verifying the broker: file path or inline PEM." }, "cert": { "type": "string", "description": "Client certificate for mTLS: file path or inline PEM." }, "key": { "type": "string", "description": "Client private key for mTLS: file path or inline PEM." }, "key_password": { "type": "string", "description": "Password protecting the client private key." }, "insecure_skip_verify": { "type": "boolean", "default": false, "description": "Disable broker certificate verification. Development/testing only." } } } } } }, "required": [ "type" ], "examples": [ { "brokers": [ "localhost:9092" ], "topics": [ "events" ], "consumer_group": "arkflow", "start_from_latest": false } ] }, { "type": "object", "title": "memory", "description": "In-memory input queue seeded with an initial list of messages. Primarily for tests and demos.", "properties": { "type": { "type": "string", "const": "memory", "description": "In-memory input queue seeded with an initial list of messages. Primarily for tests and demos." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "messages": { "type": "array", "items": { "type": "string" }, "description": "Initial messages to enqueue." } }, "required": [ "type" ], "examples": [ { "messages": [ "hello", "world" ] } ] }, { "type": "object", "title": "modbus", "description": "Polls Modbus TCP devices on a fixed interval, reading coils, discrete inputs, or registers.", "properties": { "type": { "type": "string", "const": "modbus", "description": "Polls Modbus TCP devices on a fixed interval, reading coils, discrete inputs, or registers." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "addr": { "type": "string", "description": "Modbus server address (host:port)." }, "slave_id": { "type": "integer", "description": "Modbus unit / slave ID." }, "points": { "type": "array", "description": "Points to read on every poll.", "items": { "type": "object", "properties": { "type": { "type": "string", "enum": [ "coils", "discrete_inputs", "holding_registers", "input_registers" ] }, "name": { "type": "string" }, "address": { "type": "integer" }, "quantity": { "type": "integer", "minimum": 1 } }, "required": [ "type", "name", "address", "quantity" ] } }, "interval": { "type": "string", "description": "Poll interval (humantime)." } }, "required": [ "type" ], "examples": [ { "addr": "127.0.0.1:502", "slave_id": 1, "interval": "1s", "points": [ { "type": "holding_registers", "name": "voltage", "address": 0, "quantity": 1 } ] } ] }, { "type": "object", "title": "mqtt", "description": "Subscribes to an MQTT broker and forwards messages from the configured topics.", "properties": { "type": { "type": "string", "const": "mqtt", "description": "Subscribes to an MQTT broker and forwards messages from the configured topics." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "host": { "type": "string", "description": "MQTT broker hostname." }, "port": { "type": "integer", "minimum": 1, "maximum": 65535, "description": "MQTT broker port." }, "client_id": { "type": "string", "description": "Unique client identifier." }, "username": { "type": "string", "description": "Optional username." }, "password": { "type": "string", "description": "Optional password." }, "topics": { "type": "array", "items": { "type": "string" }, "description": "Topics to subscribe to (MQTT wildcards supported)." }, "qos": { "type": "integer", "enum": [ 0, 1, 2 ], "default": 0, "description": "Quality of Service level." }, "clean_session": { "type": "boolean", "default": true, "description": "Whether to use a clean session." }, "keep_alive": { "type": "integer", "minimum": 1, "description": "Keep-alive interval in seconds." }, "tls": { "type": "object", "description": "TLS transport configuration.", "properties": { "enabled": { "type": "boolean", "default": true }, "ca": { "type": "string", "description": "CA certificate file path." }, "client_cert": { "type": "string", "description": "Client certificate file (mTLS)." }, "client_key": { "type": "string", "description": "Client private key file (mTLS)." } } } }, "required": [ "type" ], "examples": [ { "host": "localhost", "port": 1883, "client_id": "arkflow", "topics": [ "sensors/#" ] } ] }, { "type": "object", "title": "multiple_inputs", "description": "Combines multiple input sources into a single stream. Each source is tagged with __meta_source.", "properties": { "type": { "type": "string", "const": "multiple_inputs", "description": "Combines multiple input sources into a single stream. Each source is tagged with __meta_source." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "inputs": { "type": "array", "minItems": 1, "description": "List of input components to combine.", "items": { "type": "object", "properties": { "type": { "type": "string", "description": "Input type (e.g. 'kafka', 'http')." }, "name": { "type": "string", "description": "Optional logical name for this source." } }, "required": [ "type" ] } } }, "required": [ "type" ], "examples": [ { "inputs": [ { "type": "kafka", "name": "events", "brokers": [ "localhost:9092" ], "topics": [ "events" ], "consumer_group": "arkflow", "start_from_latest": false }, { "type": "kafka", "name": "logs", "brokers": [ "localhost:9092" ], "topics": [ "logs" ], "consumer_group": "arkflow", "start_from_latest": false } ] } ] }, { "type": "object", "title": "nats", "description": "Consumes messages from NATS, supporting both regular subjects and JetStream consumers.", "properties": { "type": { "type": "string", "const": "nats", "description": "Consumes messages from NATS, supporting both regular subjects and JetStream consumers." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "NATS server URL (e.g. nats://localhost:4222)." }, "mode": { "type": "object", "description": "Select between plain NATS and JetStream subscriptions.", "oneOf": [ { "properties": { "type": { "const": "regular" }, "subject": { "type": "string" }, "queue_group": { "type": "string" } }, "required": [ "type", "subject" ] }, { "properties": { "type": { "const": "jet_stream" }, "stream": { "type": "string" }, "consumer_name": { "type": "string" }, "durable_name": { "type": "string" } }, "required": [ "type", "stream", "consumer_name" ] } ] }, "auth": { "type": "object", "description": "NATS authentication configuration." } }, "required": [ "type" ], "examples": [ { "url": "nats://localhost:4222", "mode": { "type": "regular", "subject": "events" } } ] }, { "type": "object", "title": "pulsar", "description": "Subscribes to an Apache Pulsar topic with configurable subscription type and authentication.", "properties": { "type": { "type": "string", "const": "pulsar", "description": "Subscribes to an Apache Pulsar topic with configurable subscription type and authentication." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "service_url": { "type": "string", "description": "Pulsar service URL (e.g. pulsar://localhost:6650)." }, "topic": { "type": "string", "description": "Topic to subscribe to." }, "subscription_name": { "type": "string", "description": "Subscription name." }, "subscription_type": { "type": "string", "enum": [ "Exclusive", "Shared", "Failover", "KeyShared" ], "description": "Subscription type." }, "auth": { "type": "object", "description": "Pulsar authentication configuration." }, "retry_config": { "type": "object", "description": "Retry behaviour for failed messages." } }, "required": [ "type" ], "examples": [ { "service_url": "pulsar://localhost:6650", "topic": "persistent://public/default/events", "subscription_name": "arkflow" } ] }, { "type": "object", "title": "redis", "description": "Reads from Redis: list blocking pops, pub/sub subscriptions, or stream consumer groups.", "properties": { "type": { "type": "string", "const": "redis", "description": "Reads from Redis: list blocking pops, pub/sub subscriptions, or stream consumer groups." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "mode": { "type": "object", "description": "Connection mode.", "oneOf": [ { "properties": { "type": { "const": "cluster" }, "urls": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "urls" ] }, { "properties": { "type": { "const": "single" }, "url": { "type": "string" } }, "required": [ "type", "url" ] } ] }, "redis_type": { "type": "object", "description": "Data structure to consume from.", "oneOf": [ { "properties": { "type": { "const": "subscribe" }, "subscribe": { "type": "object", "oneOf": [ { "properties": { "type": { "const": "channels" }, "channels": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "channels" ] }, { "properties": { "type": { "const": "patterns" }, "patterns": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "patterns" ] } ] } }, "required": [ "type", "subscribe" ] }, { "properties": { "type": { "const": "list" }, "list": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "list" ] } ] } }, "required": [ "type" ], "examples": [ { "mode": { "type": "single", "url": "redis://localhost:6379" }, "redis_type": { "type": "list", "list": [ "events" ] } } ] }, { "type": "object", "title": "sql", "description": "Runs a one-shot SELECT against a SQL database (MySQL / PostgreSQL / SQLite / DuckDB) and emits the rows as batches.", "properties": { "type": { "type": "string", "const": "sql", "description": "Runs a one-shot SELECT against a SQL database (MySQL / PostgreSQL / SQLite / DuckDB) and emits the rows as batches." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "select_sql": { "type": "string", "description": "SELECT statement executed once at startup." }, "ballista": { "type": "object", "description": "Optional Ballista distributed compute configuration." }, "input_type": { "type": "object", "description": "Database connection settings." } }, "required": [ "type" ], "examples": [ { "select_sql": "SELECT id, name FROM users", "input_type": { "type": "sqlite", "name": "users", "path": "./data.db" } } ] }, { "type": "object", "title": "websocket", "description": "Connects to a WebSocket server and forwards each incoming message as a batch.", "properties": { "type": { "type": "string", "const": "websocket", "description": "Connects to a WebSocket server and forwards each incoming message as a batch." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "WebSocket server URL (ws:// or wss://)." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Headers included in the WebSocket handshake." }, "timeout": { "type": "integer", "minimum": 1, "description": "Connection timeout in seconds." } }, "required": [ "type" ], "examples": [ { "url": "ws://localhost:8080/stream" } ] } ] }, "output": { "type": "object", "description": "Select a registered output component.", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ null, null, null, null, null, null, null, null, null, null, null, null, null, null ], "description": "output component type." }, "name": { "type": "string", "description": "Optional logical name for this component instance." } }, "oneOf": [ { "type": "object", "title": "drop", "description": "Discards all messages. Useful for performance benchmarks and dead-end pipelines.", "properties": { "type": { "type": "string", "const": "drop", "description": "Discards all messages. Useful for performance benchmarks and dead-end pipelines." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] } }, "required": [ "type" ] }, { "type": "object", "title": "http", "description": "Posts each batch to an HTTP endpoint. Supports custom headers, retry, and auth.", "properties": { "type": { "type": "string", "const": "http", "description": "Posts each batch to an HTTP endpoint. Supports custom headers, retry, and auth." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "Destination URL." }, "method": { "type": "string", "enum": [ "GET", "POST", "PUT", "DELETE", "PATCH" ], "default": "POST", "description": "HTTP method." }, "timeout_ms": { "type": "integer", "minimum": 1, "description": "Request timeout in milliseconds." }, "retry_count": { "type": "integer", "minimum": 0, "description": "Number of retry attempts on failure." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Custom HTTP headers." }, "body_field": { "type": "string", "description": "Record field that holds the request body." }, "auth": { "type": "object", "description": "Authentication configuration." } }, "required": [ "type" ], "examples": [ { "url": "https://example.com/ingest", "method": "POST", "timeout_ms": 5000, "retry_count": 3 } ] }, { "type": "object", "title": "influxdb", "description": "Writes time-series data to InfluxDB v2.x using the Line Protocol.", "properties": { "type": { "type": "string", "const": "influxdb", "description": "Writes time-series data to InfluxDB v2.x using the Line Protocol." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "InfluxDB server URL (e.g. http://localhost:8086)." }, "org": { "type": "string", "description": "Organization name." }, "bucket": { "type": "string", "description": "Destination bucket." }, "token": { "type": "string", "description": "Authentication token." }, "measurement": { "type": "string", "description": "Measurement name." }, "tags": { "type": "array", "items": { "type": "object", "properties": { "field": { "type": "string" }, "tag_name": { "type": "string" } }, "required": [ "field", "tag_name" ] }, "description": "Tag mappings (label fields)." }, "fields": { "type": "array", "items": { "type": "object", "properties": { "field": { "type": "string" }, "field_name": { "type": "string" }, "field_type": { "enum": [ "float", "integer", "boolean", "string" ] } }, "required": [ "field", "field_name" ] }, "description": "Field mappings (value fields)." }, "timestamp_field": { "type": "string", "description": "Source field for the point timestamp." }, "batch_size": { "type": "integer", "minimum": 1, "description": "Batch size for write requests." }, "flush_interval": { "type": "integer", "minimum": 1, "description": "Maximum seconds to wait before flushing a partial batch." }, "retry_count": { "type": "integer", "minimum": 1, "description": "Number of HTTP attempts per flush." }, "timeout_ms": { "type": "integer", "minimum": 1, "description": "HTTP request timeout in milliseconds." } }, "required": [ "type" ], "examples": [ { "url": "http://localhost:8086", "org": "arkflow", "bucket": "metrics", "token": "${INFLUX_TOKEN}", "measurement": "sensor", "tags": [ { "field": "sensor", "tag_name": "sensor" } ], "fields": [ { "field": "value", "field_name": "value", "field_type": "float" } ], "timestamp_field": "timestamp" } ] }, { "type": "object", "title": "kafka", "description": "Produces messages to Apache Kafka. Supports key-based partitioning and compression.", "properties": { "type": { "type": "string", "const": "kafka", "description": "Produces messages to Apache Kafka. Supports key-based partitioning and compression." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "brokers": { "type": "array", "items": { "type": "string" }, "description": "List of Kafka broker addresses." }, "topic": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ], "description": "Literal value or a SQL expression evaluated per batch." }, "key": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ], "description": "Literal value or a SQL expression evaluated per batch." }, "client_id": { "type": "string", "description": "Optional client identifier." }, "compression": { "type": "string", "enum": [ "none", "gzip", "snappy", "lz4", "zstd" ], "description": "Compression algorithm." }, "acks": { "type": "string", "enum": [ "0", "1", "all" ], "description": "Acknowledgment level." }, "value_field": { "type": "string", "description": "Record field used as the message payload." }, "exactly_once": { "type": "boolean", "default": false, "description": "Enable exactly-once transactional production (L2)." }, "transactional_id": { "type": "string", "description": "Transactional id (required when exactly_once is true); must be stable across restarts for zombie fencing." }, "offset_commit_group": { "type": "string", "description": "L3 exactly-once: consumer group of a paired Kafka input (with transactional_offsets: true) whose source offsets commit inside this output's producer transactions. Requires exactly_once." }, "security": { "type": "object", "description": "SASL authentication and TLS settings. Omit entirely for plaintext.", "properties": { "protocol": { "type": "string", "enum": [ "plaintext", "ssl", "sasl_plaintext", "sasl_ssl" ], "description": "Security protocol. Defaults to inferred from which sub-blocks are present: both -> sasl_ssl, only sasl -> sasl_plaintext, only tls -> ssl, neither -> plaintext." }, "sasl": { "type": "object", "description": "SASL credentials (PLAIN or SCRAM).", "properties": { "mechanism": { "type": "string", "enum": [ "plain", "scram-sha-256", "scram-sha-512" ], "description": "SASL mechanism." }, "username": { "type": "string", "description": "SASL username (required for plain/scram)." }, "password": { "type": "string", "description": "SASL password (required for plain/scram)." } }, "required": [ "mechanism" ] }, "tls": { "type": "object", "description": "TLS settings. ca/cert/key accept a file path or inline PEM text (auto-detected).", "properties": { "ca": { "type": "string", "description": "CA certificate for verifying the broker: file path or inline PEM." }, "cert": { "type": "string", "description": "Client certificate for mTLS: file path or inline PEM." }, "key": { "type": "string", "description": "Client private key for mTLS: file path or inline PEM." }, "key_password": { "type": "string", "description": "Password protecting the client private key." }, "insecure_skip_verify": { "type": "boolean", "default": false, "description": "Disable broker certificate verification. Development/testing only." } } } } } }, "required": [ "type" ], "examples": [ { "brokers": [ "localhost:9092" ], "topic": { "type": "value", "value": "events" } } ] }, { "type": "object", "title": "milvus", "description": "Upserts batch rows into a Milvus collection over the REST v2 vectordb API: a Float32 list column becomes the vector, other columns pack into a JSON payload field, and an optional id column keys the row.", "properties": { "type": { "type": "string", "const": "milvus", "description": "Upserts batch rows into a Milvus collection over the REST v2 vectordb API: a Float32 list column becomes the vector, other columns pack into a JSON payload field, and an optional id column keys the row." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "Milvus base URL, e.g. http://localhost:19530." }, "collection": { "type": "string", "description": "Target collection name (schema must already exist)." }, "vector_field": { "type": "string", "description": "Name of the vector field (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "id_field": { "type": "string", "description": "Field carrying the row id (integer or string). Required: the upsert API needs a primary key per row (auto-id collections are not supported)." }, "payload_field": { "type": "string", "description": "JSON field receiving every remaining column as a per-row object. Defaults to 'payload'; set to an empty string to disable." }, "api_key": { "type": "string", "description": "Sent as 'Authorization: Bearer' (Milvus convention: :); supports secret references." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "retry_count": { "type": "integer", "description": "Retry attempts for connection errors and 5xx responses. Defaults to 0." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." } }, "required": [ "type" ], "examples": [ { "url": "http://localhost:19530", "collection": "documents", "vector_field": "embedding", "id_field": "doc_id", "api_key": "${env:MILVUS_CREDENTIALS}" } ] }, { "type": "object", "title": "mongodb", "description": "Writes Arrow rows to MongoDB as BSON documents.", "properties": { "type": { "type": "string", "const": "mongodb", "description": "Writes Arrow rows to MongoDB as BSON documents." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "uri": { "type": "string", "description": "MongoDB connection URI." }, "database": { "type": "string", "description": "Target database name." }, "collection": { "type": "string", "description": "Target collection name." } }, "required": [ "type" ], "examples": [ { "uri": "mongodb://localhost:27017", "database": "arkflow", "collection": "events" } ] }, { "type": "object", "title": "mqtt", "description": "Publishes messages to an MQTT broker topic.", "properties": { "type": { "type": "string", "const": "mqtt", "description": "Publishes messages to an MQTT broker topic." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "host": { "type": "string", "description": "MQTT broker hostname." }, "port": { "type": "integer", "minimum": 1, "maximum": 65535, "description": "MQTT broker port." }, "client_id": { "type": "string", "description": "Client identifier." }, "username": { "type": "string", "description": "Optional username." }, "password": { "type": "string", "description": "Optional password." }, "topic": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ], "description": "Literal value or a SQL expression evaluated per batch." }, "qos": { "type": "integer", "enum": [ 0, 1, 2 ], "default": 0, "description": "Quality of Service." }, "clean_session": { "type": "boolean", "default": true }, "keep_alive": { "type": "integer", "minimum": 1, "description": "Keep-alive interval in seconds." }, "retain": { "type": "boolean", "default": false, "description": "Whether to retain the message on the broker." }, "value_field": { "type": "string", "description": "Column whose per-row value becomes the message payload (binary or string column). When unset, the codec encoding of the batch applies." }, "tls": { "type": "object", "description": "TLS transport configuration.", "properties": { "enabled": { "type": "boolean", "default": true }, "ca": { "type": "string", "description": "CA certificate file path." }, "client_cert": { "type": "string", "description": "Client certificate file (mTLS)." }, "client_key": { "type": "string", "description": "Client private key file (mTLS)." } } } }, "required": [ "type" ], "examples": [ { "host": "localhost", "port": 1883, "client_id": "arkflow", "topic": { "type": "value", "value": "sensors/data" } } ] }, { "type": "object", "title": "nats", "description": "Publishes to NATS, either to a regular subject or a JetStream stream.", "properties": { "type": { "type": "string", "const": "nats", "description": "Publishes to NATS, either to a regular subject or a JetStream stream." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "NATS server URL." }, "mode": { "type": "object", "description": "Select regular or JetStream publishing.", "oneOf": [ { "properties": { "type": { "const": "regular" }, "subject": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "subject" ] }, { "properties": { "type": { "const": "jet_stream" }, "stream": { "type": "string" }, "subject": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "stream", "subject" ] } ] }, "auth": { "type": "object", "description": "NATS authentication configuration." }, "value_field": { "type": "string", "description": "Record field used as the payload." } }, "required": [ "type" ], "examples": [ { "url": "nats://localhost:4222", "mode": { "type": "regular", "subject": { "type": "value", "value": "events" } } } ] }, { "type": "object", "title": "pgvector", "description": "Upserts batch rows into a PostgreSQL table with the pgvector extension: a Float32 list column becomes the vector, other columns are packed into a jsonb payload, and an optional id column keys ON CONFLICT upserts.", "properties": { "type": { "type": "string", "const": "pgvector", "description": "Upserts batch rows into a PostgreSQL table with the pgvector extension: a Float32 list column becomes the vector, other columns are packed into a jsonb payload, and an optional id column keys ON CONFLICT upserts." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "Postgres connection string, e.g. postgres://user:pass@host:5432/db; supports secret references." }, "table": { "type": "string", "description": "Target table (must already exist with a matching vector column)." }, "vector_field": { "type": "string", "description": "Name of the vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "id_field": { "type": "string", "description": "Column used as the upsert conflict key (integer or string). When omitted, plain INSERTs are written." }, "payload_field": { "type": "string", "description": "jsonb column receiving every remaining column as a per-row JSON object. Defaults to 'payload'; set to an empty string to disable." }, "max_connections": { "type": "integer", "description": "Connection pool size. Defaults to 4." }, "timeout_ms": { "type": "integer", "description": "Pool acquire/connect timeout in milliseconds. Defaults to 30000." } }, "required": [ "type" ], "examples": [ { "url": "postgres://postgres:postgres@localhost:5432/vectors", "table": "documents", "vector_field": "embedding", "id_field": "doc_id" } ] }, { "type": "object", "title": "pulsar", "description": "Produces messages to an Apache Pulsar topic.", "properties": { "type": { "type": "string", "const": "pulsar", "description": "Produces messages to an Apache Pulsar topic." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "service_url": { "type": "string", "description": "Pulsar service URL." }, "topic": { "type": "string", "description": "Destination topic (supports {field} placeholders)." }, "auth": { "type": "object", "description": "Pulsar authentication configuration." }, "value_field": { "type": "string", "description": "Record field used as the payload." } }, "required": [ "type" ], "examples": [ { "service_url": "pulsar://localhost:6650", "topic": "persistent://public/default/events" } ] }, { "type": "object", "title": "qdrant", "description": "Upserts batch rows as Qdrant points over the REST API: a vector column, an optional id column, and all remaining columns as payload.", "properties": { "type": { "type": "string", "const": "qdrant", "description": "Upserts batch rows as Qdrant points over the REST API: a vector column, an optional id column, and all remaining columns as payload." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "Qdrant base URL, e.g. http://localhost:6333." }, "collection": { "type": "string", "description": "Target collection name." }, "vector_field": { "type": "string", "description": "Name of the vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "id_field": { "type": "string", "description": "Column used as the point id (unsigned integer or string). When omitted, Qdrant generates ids." }, "payload_fields": { "type": "array", "items": { "type": "string" }, "description": "Columns to include in the point payload. Defaults to every column except the vector and id columns." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer'; supports secret references." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "retry_count": { "type": "integer", "description": "Retry attempts for connection errors and 5xx responses. Defaults to 3." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." } }, "required": [ "type" ], "examples": [ { "url": "http://localhost:6333", "collection": "documents", "vector_field": "embedding", "id_field": "doc_id" } ] }, { "type": "object", "title": "redis", "description": "Writes messages to Redis via publish, list rpush, hashes, or plain strings.", "properties": { "type": { "type": "string", "const": "redis", "description": "Writes messages to Redis via publish, list rpush, hashes, or plain strings." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "mode": { "type": "object", "description": "Connection mode (single or cluster).", "oneOf": [ { "properties": { "type": { "const": "cluster" }, "urls": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "urls" ] }, { "properties": { "type": { "const": "single" }, "url": { "type": "string" } }, "required": [ "type", "url" ] } ] }, "redis_type": { "type": "object", "description": "Destination data structure.", "oneOf": [ { "properties": { "type": { "const": "publish" }, "channel": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "channel" ] }, { "properties": { "type": { "const": "list" }, "key": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "key" ] }, { "properties": { "type": { "const": "hashes" }, "key": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] }, "field": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "key", "field" ] }, { "properties": { "type": { "const": "strings" }, "key": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "key" ] } ] }, "value_field": { "type": "string", "description": "Record field used as the payload." } }, "required": [ "type" ], "examples": [ { "mode": { "type": "single", "url": "redis://localhost:6379" }, "redis_type": { "type": "publish", "channel": { "type": "value", "value": "events" } } } ] }, { "type": "object", "title": "sql", "description": "Batch-inserts records into a MySQL or PostgreSQL database, with optional upsert (ON DUPLICATE KEY UPDATE / ON CONFLICT DO UPDATE) for idempotent writes.", "properties": { "type": { "type": "string", "const": "sql", "description": "Batch-inserts records into a MySQL or PostgreSQL database, with optional upsert (ON DUPLICATE KEY UPDATE / ON CONFLICT DO UPDATE) for idempotent writes." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "output_type": { "type": "object", "description": "Database connection settings, tagged by `type` (`mysql` or `postgres`), each with `uri` and optional `ssl`.", "properties": { "type": { "type": "string", "enum": [ "mysql", "postgres" ] }, "uri": { "type": "string" }, "ssl": { "type": "object", "description": "Optional SSL configuration (ssl_mode, root_cert, client_cert, client_key)." } }, "required": [ "type", "uri" ] }, "table_name": { "type": "string", "description": "Destination table." }, "upsert": { "type": "boolean", "default": false, "description": "Use upsert (MySQL ON DUPLICATE KEY UPDATE / PostgreSQL ON CONFLICT DO UPDATE) instead of a plain insert." }, "upsert_keys": { "type": "array", "items": { "type": "string" }, "description": "Columns used as the conflict target for upsert. Required (non-empty) when upsert is true." } }, "required": [ "type" ], "examples": [ { "output_type": { "type": "postgres", "uri": "postgres://user:pass@localhost/db" }, "table_name": "events", "upsert": true, "upsert_keys": [ "id" ] } ] }, { "type": "object", "title": "stdout", "description": "Writes each message to the console. Useful for debugging and demos.", "properties": { "type": { "type": "string", "const": "stdout", "description": "Writes each message to the console. Useful for debugging and demos." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "append_newline": { "type": "boolean", "default": true, "description": "Append a newline after each message." } }, "required": [ "type" ], "examples": [ { "append_newline": true } ] } ] }, "processor": { "type": "object", "description": "Select a registered processor component.", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ null, null, null, null, null, null, null, null, null, null, null, null, null ], "description": "processor component type." }, "name": { "type": "string", "description": "Optional logical name for this component instance." } }, "oneOf": [ { "type": "object", "title": "arrow_to_json", "description": "Converts an Arrow RecordBatch into JSON byte payloads (one per row).", "properties": { "type": { "type": "string", "const": "arrow_to_json", "description": "Converts an Arrow RecordBatch into JSON byte payloads (one per row)." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "value_field": { "type": "string", "description": "Binary column carrying one JSON payload per row (defaults to the standard binary value column)." }, "fields_to_include": { "type": "array", "items": { "type": "string" }, "description": "Only emit these top-level fields." } }, "required": [ "type" ] }, { "type": "object", "title": "arrow_to_protobuf", "description": "Serializes Arrow RecordBatches into Protobuf wire-format bytes.", "properties": { "type": { "type": "string", "const": "arrow_to_protobuf", "description": "Serializes Arrow RecordBatches into Protobuf wire-format bytes." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "message_type": { "type": "string", "description": "Fully-qualified Protobuf message type name." }, "proto_inputs": { "type": "array", "items": { "type": "string" }, "description": "Paths to .proto files." }, "proto_includes": { "type": "array", "items": { "type": "string" }, "description": "Include paths for proto resolution." }, "fields_to_include": { "type": "array", "items": { "type": "string" }, "description": "Optional allow-list of field names to include when serializing to Protobuf." } }, "required": [ "type" ] }, { "type": "object", "title": "batch", "description": "Batches messages by row count with an idle timeout before forwarding.", "properties": { "type": { "type": "string", "const": "batch", "description": "Batches messages by row count with an idle timeout before forwarding." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "count": { "type": "integer", "minimum": 1, "description": "Number of message rows to accumulate before flushing." }, "timeout_ms": { "type": "integer", "minimum": 1, "description": "Idle timeout that flushes a partial batch (milliseconds)." } }, "required": [ "type" ], "examples": [ { "count": 100, "timeout_ms": 5000 } ] }, { "type": "object", "title": "embedding", "description": "Batch-embeds a text column through an OpenAI-compatible embeddings API and appends the vectors as a FixedSizeList(Float32) column.", "properties": { "type": { "type": "string", "const": "embedding", "description": "Batch-embeds a text column through an OpenAI-compatible embeddings API and appends the vectors as a FixedSizeList(Float32) column." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "api_base": { "type": "string", "description": "Base URL of the embeddings API, e.g. https://api.openai.com/v1 (the request path is {api_base}/embeddings)." }, "model": { "type": "string", "description": "Embedding model name, e.g. text-embedding-3-small." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer'; supports secret references. Omit for unauthenticated endpoints." }, "field": { "type": "string", "description": "Name of the input UTF-8 column to embed." }, "target_field": { "type": "string", "description": "Name of the appended vector column. Defaults to 'embedding'." }, "batch_size": { "type": "integer", "description": "Maximum rows per HTTP request. Defaults to 32." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers, e.g. Azure's 'api-key'." } }, "required": [ "type" ], "examples": [ { "api_base": "https://api.openai.com/v1", "model": "text-embedding-3-small", "api_key": "${env:OPENAI_API_KEY}", "field": "text", "batch_size": 32 } ] }, { "type": "object", "title": "json_to_arrow", "description": "Parses JSON byte payloads into an Arrow RecordBatch with inferred schema.", "properties": { "type": { "type": "string", "const": "json_to_arrow", "description": "Parses JSON byte payloads into an Arrow RecordBatch with inferred schema." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "value_field": { "type": "string", "description": "Binary column carrying one JSON payload per row (defaults to the standard binary value column)." }, "fields_to_include": { "type": "array", "items": { "type": "string" }, "description": "Only parse these top-level fields into columns." } }, "required": [ "type" ] }, { "type": "object", "title": "llm", "description": "Sends each row of a text column to an OpenAI-compatible chat completions API and appends the completion as a Utf8 column, with bounded ordered concurrency.", "properties": { "type": { "type": "string", "const": "llm", "description": "Sends each row of a text column to an OpenAI-compatible chat completions API and appends the completion as a Utf8 column, with bounded ordered concurrency." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "api_base": { "type": "string", "description": "Base URL of the API, e.g. https://api.openai.com/v1 (the request path is {api_base}/chat/completions)." }, "model": { "type": "string", "description": "Chat model name, e.g. gpt-4o-mini." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer'; supports secret references. Omit for unauthenticated endpoints." }, "field": { "type": "string", "description": "Name of the input UTF-8 column to send." }, "target_field": { "type": "string", "description": "Name of the appended completion column. Defaults to 'response'." }, "system_prompt": { "type": "string", "description": "Optional system message prepended to each request." }, "prompt_template": { "type": "string", "description": "Optional user-message template; '{{value}}' is replaced with the row text. When omitted the row text is the user message." }, "temperature": { "type": "number", "description": "Sampling temperature; omitted from the request when unset." }, "max_tokens": { "type": "integer", "description": "Completion token cap; omitted from the request when unset." }, "concurrency": { "type": "integer", "description": "Maximum in-flight requests. Defaults to 4." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." }, "stream": { "type": "boolean", "description": "Request SSE streaming (`stream: true`) and assemble the deltas into the row's final completion text. Useful behind providers or gateways that time out non-streaming long completions." }, "tools": { "description": "OpenAI tools definition passed through verbatim to the request body for function calling." }, "tool_calls_column": { "type": "string", "description": "When set, appends this column with the response's tool_calls as JSON text (empty string when the response has none). Streamed tool_call deltas are merged by index." } }, "required": [ "type" ], "examples": [ { "api_base": "https://api.openai.com/v1", "model": "gpt-4o-mini", "api_key": "${env:OPENAI_API_KEY}", "field": "text", "system_prompt": "You classify support tickets with one word.", "concurrency": 4 } ] }, { "type": "object", "title": "milvus_search", "description": "Searches a Milvus collection for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column. Rows are searched with bounded, order-preserving concurrency.", "properties": { "type": { "type": "string", "const": "milvus_search", "description": "Searches a Milvus collection for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column. Rows are searched with bounded, order-preserving concurrency." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "Milvus base URL, e.g. http://localhost:19530." }, "collection": { "type": "string", "description": "Collection to search." }, "vector_field": { "type": "string", "description": "Name of the query vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "target_field": { "type": "string", "description": "Name of the appended matches column (JSON array text). Defaults to 'matches'." }, "id_field": { "type": "string", "description": "Collection field returned as the match id. When omitted the id key is left out (auto-id collections)." }, "payload_field": { "type": "string", "description": "Collection field included in each match as a payload object. Defaults to 'payload'; set to an empty string to disable." }, "metric": { "type": "string", "enum": [ "COSINE", "L2", "IP" ], "description": "Metric type; must match the collection schema. Defaults to COSINE." }, "top_k": { "type": "integer", "description": "Number of neighbors per row. Defaults to 5." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer' (Milvus convention: :); supports secret references." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." } }, "required": [ "type" ], "examples": [ { "url": "http://localhost:19530", "collection": "documents", "vector_field": "embedding", "id_field": "doc_id", "top_k": 5 } ] }, { "type": "object", "title": "pgvector_search", "description": "Searches a PostgreSQL table with the pgvector extension for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column.", "properties": { "type": { "type": "string", "const": "pgvector_search", "description": "Searches a PostgreSQL table with the pgvector extension for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "Postgres connection string; supports secret references." }, "table": { "type": "string", "description": "Table to search (must already exist with a pgvector column)." }, "vector_field": { "type": "string", "description": "Name of the query vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "target_field": { "type": "string", "description": "Name of the appended matches column (JSON array text). Defaults to 'matches'." }, "id_column": { "type": "string", "description": "Table column returned as the match id (as text). Defaults to 'id'." }, "vector_column": { "type": "string", "description": "Table column holding the stored vectors. Defaults to 'embedding'." }, "payload_column": { "type": "string", "description": "jsonb column included in each match as a payload object. Defaults to 'payload'; set to an empty string to disable." }, "metric": { "type": "string", "enum": [ "cosine", "l2", "inner_product" ], "description": "Distance operator: <=> (cosine, default), <-> (l2), <#> (inner product, negative)." }, "top_k": { "type": "integer", "description": "Number of neighbors per row. Defaults to 5." }, "concurrency": { "type": "integer", "description": "Maximum in-flight queries. Defaults to 4." }, "max_connections": { "type": "integer", "description": "Connection pool size. Defaults to 4." }, "timeout_ms": { "type": "integer", "description": "Pool acquire timeout in milliseconds. Defaults to 30000." } }, "required": [ "type" ], "examples": [ { "url": "postgres://postgres:postgres@localhost:5432/vectors", "table": "documents", "metric": "cosine", "top_k": 5 } ] }, { "type": "object", "title": "protobuf_to_arrow", "description": "Decodes Protobuf wire-format bytes into Arrow RecordBatches.", "properties": { "type": { "type": "string", "const": "protobuf_to_arrow", "description": "Decodes Protobuf wire-format bytes into Arrow RecordBatches." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "message_type": { "type": "string", "description": "Fully-qualified Protobuf message type name." }, "proto_inputs": { "type": "array", "items": { "type": "string" }, "description": "Paths to .proto files." }, "proto_includes": { "type": "array", "items": { "type": "string" }, "description": "Include paths for proto resolution." }, "value_field": { "type": "string", "description": "Name of the binary column holding the Protobuf wire-format bytes (defaults to '__value')." } }, "required": [ "type" ] }, { "type": "object", "title": "python", "description": "Runs a user-defined Python function (with PyArrow) against each batch.", "properties": { "type": { "type": "string", "const": "python", "description": "Runs a user-defined Python function (with PyArrow) against each batch." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "script": { "type": "string", "description": "Python source defining the transform function (optional when importing an existing module)." }, "module": { "type": "string", "default": "__main__", "description": "Python module providing the function." }, "function": { "type": "string", "description": "Name of the function to invoke for each batch." }, "python_path": { "type": "array", "items": { "type": "string" }, "default": [], "description": "Extra sys.path entries, earlier entries take precedence." }, "timeout_ms": { "type": "integer", "minimum": 1, "default": 60000, "description": "Upper bound for one UDF call in milliseconds; longer calls fail the batch." } }, "required": [ "type" ], "examples": [ { "script": "def transform(batch):\n return batch", "function": "transform" } ] }, { "type": "object", "title": "sql", "description": "Runs a DataFusion SQL query against each batch. Supports window functions and joins against temporary tables.", "properties": { "type": { "type": "string", "const": "sql", "description": "Runs a DataFusion SQL query against each batch. Supports window functions and joins against temporary tables." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "query": { "type": "string", "description": "SQL query to run on every batch." }, "table_name": { "type": "string", "description": "Name used for the batch table in the query (default 'flow')." }, "temporary_list": { "type": "array", "description": "Temporary tables to register before running the query.", "items": { "type": "object", "properties": { "name": { "type": "string" }, "table_name": { "type": "string" }, "key": { "type": "object", "properties": { "value": { "type": "string" } }, "required": [ "value" ] } }, "required": [ "name", "table_name", "key" ] } } }, "required": [ "type" ], "examples": [ { "query": "SELECT *, __meta_source AS source FROM flow" } ] }, { "type": "object", "title": "vector_search", "description": "Searches a Qdrant collection for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column.", "properties": { "type": { "type": "string", "const": "vector_search", "description": "Searches a Qdrant collection for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "url": { "type": "string", "description": "Qdrant base URL, e.g. http://localhost:6333." }, "collection": { "type": "string", "description": "Collection to search." }, "vector_field": { "type": "string", "description": "Name of the query vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "target_field": { "type": "string", "description": "Name of the appended matches column (JSON array text). Defaults to 'matches'." }, "top_k": { "type": "integer", "description": "Number of neighbors per row. Defaults to 5." }, "score_threshold": { "type": "number", "description": "Minimum similarity score; omitted from the request when unset." }, "concurrency": { "type": "integer", "description": "Maximum in-flight requests. Defaults to 4." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer'; supports secret references." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." } }, "required": [ "type" ], "examples": [ { "url": "http://localhost:6333", "collection": "documents", "vector_field": "embedding", "top_k": 5 } ] }, { "type": "object", "title": "vrl", "description": "Runs a Vector Remap Language (VRL) program against each batch for safe transformation and enrichment.", "properties": { "type": { "type": "string", "const": "vrl", "description": "Runs a Vector Remap Language (VRL) program against each batch for safe transformation and enrichment." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "statement": { "type": "string", "description": "VRL program source." }, "timezone": { "type": "string", "description": "Optional timezone for VRL timestamp operations (e.g. 'Asia/Shanghai', 'UTC', 'local'). Defaults to the platform local timezone; invalid values fall back to the default with a warning." } }, "required": [ "type" ], "examples": [ { "statement": ".message = parse_json!(.message)\n.timestamp = now()" } ] } ] }, "buffer": { "type": "object", "description": "Select a registered buffer component.", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ null, null, null, null ], "description": "buffer component type." }, "name": { "type": "string", "description": "Optional logical name for this component instance." } }, "oneOf": [ { "type": "object", "title": "memory", "description": "In-memory buffer that releases a batch when it reaches capacity or after a timeout.", "properties": { "type": { "type": "string", "const": "memory", "description": "In-memory buffer that releases a batch when it reaches capacity or after a timeout." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "capacity": { "type": "integer", "minimum": 1, "description": "Maximum number of messages to accumulate before releasing." }, "timeout": { "type": "string", "description": "Maximum time to wait before releasing a partial batch (humantime)." } }, "required": [ "type" ], "examples": [ { "capacity": 1000, "timeout": "5s" } ] }, { "type": "object", "title": "session_window", "description": "Groups messages into sessions based on a maximum gap between messages. Supports SQL joins across sources.", "properties": { "type": { "type": "string", "const": "session_window", "description": "Groups messages into sessions based on a maximum gap between messages. Supports SQL joins across sources." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "gap": { "type": "string", "description": "Maximum idle time before a session is closed (humantime)." }, "join": { "type": "object", "description": "Optional SQL join across input sources.", "properties": { "query": { "type": "string" }, "value_field": { "type": "string" }, "codec": { "type": "object" }, "thread_num": { "type": "integer", "minimum": 1 } } } }, "required": [ "type" ], "examples": [ { "gap": "30s" } ] }, { "type": "object", "title": "sliding_window", "description": "Overlapping time windows that slide forward by a fixed interval.", "properties": { "type": { "type": "string", "const": "sliding_window", "description": "Overlapping time windows that slide forward by a fixed interval." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "window_size": { "type": "integer", "minimum": 1, "description": "Number of messages included in each window." }, "interval": { "type": "string", "description": "Time interval between window emissions (humantime)." }, "slide_size": { "type": "integer", "minimum": 1, "description": "Number of messages to advance the window by on each emission." } }, "required": [ "type" ], "examples": [ { "window_size": 100, "interval": "5s", "slide_size": 20 } ] }, { "type": "object", "title": "tumbling_window", "description": "Fixed-size, non-overlapping time windows. Supports SQL joins across sources.", "properties": { "type": { "type": "string", "const": "tumbling_window", "description": "Fixed-size, non-overlapping time windows. Supports SQL joins across sources." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "interval": { "type": "string", "description": "Window duration (humantime)." }, "join": { "type": "object", "description": "Optional SQL join across input sources.", "properties": { "query": { "type": "string" }, "value_field": { "type": "string" }, "codec": { "type": "object" }, "thread_num": { "type": "integer", "minimum": 1 } } } }, "required": [ "type" ], "examples": [ { "interval": "1m" } ] } ] }, "codec": { "type": "object", "description": "Select a registered codec component.", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ null, null, null, null ], "description": "codec component type." }, "name": { "type": "string", "description": "Optional logical name for this component instance." } }, "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "on_error": { "type": "string", "enum": [ "fail", "skip" ], "default": "fail", "description": "Decode-error policy: `fail` fails the batch (default), `skip` isolates bad messages with a warning and decodes the rest." } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "message_type": { "type": "string", "description": "Fully-qualified Protobuf message type name." }, "proto_inputs": { "type": "array", "items": { "type": "string" }, "description": "Paths to .proto files." }, "proto_includes": { "type": "array", "items": { "type": "string" }, "description": "Include paths for proto resolution." }, "on_error": { "type": "string", "enum": [ "fail", "skip" ], "default": "fail", "description": "Decode-error policy: `fail` fails the batch (default), `skip` isolates bad messages with a warning and decodes the rest." } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "oneOf": [ { "type": "object", "title": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "properties": { "type": { "type": "string", "const": "debezium_json" } }, "required": [ "type" ] }, { "type": "object", "title": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "properties": { "type": { "type": "string", "const": "json" } }, "required": [ "type" ] }, { "type": "object", "title": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "properties": { "type": { "type": "string", "const": "protobuf" } }, "required": [ "type" ] }, { "type": "object", "title": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "properties": { "type": { "type": "string", "const": "schema_registry" } }, "required": [ "type" ] } ] }, "registry_url": { "type": "string", "description": "Confluent Schema Registry base URL." }, "message_type": { "type": "string", "description": "Fully-qualified Protobuf message type. Required for Protobuf schemas; omit for Avro." }, "subject": { "type": "string", "description": "Registry subject for the compatibility gate." }, "min_compatibility": { "type": "string", "enum": [ "none", "backward", "forward", "full" ], "description": "Minimum subject compatibility level enforced on the first decoded message. Requires `subject`." }, "auth": { "type": "object", "description": "Optional registry authentication.", "properties": { "type": { "type": "string", "enum": [ "basic", "bearer" ] }, "username": { "type": "string" }, "password": { "type": "string" }, "token": { "type": "string" } } } }, "required": [ "type" ] } ] }, "temporary": { "type": "object", "description": "Select a registered temporary component.", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ null ], "description": "temporary component type." }, "name": { "type": "string", "description": "Optional logical name for this component instance." } }, "oneOf": [ { "type": "object", "title": "redis", "description": "Redis-backed temporary lookup store (single node or cluster) read through a codec.", "properties": { "type": { "type": "string", "const": "redis", "description": "Redis-backed temporary lookup store (single node or cluster) read through a codec." }, "name": { "type": "string", "description": "Optional logical name for this instance." }, "codec": { "type": "object", "description": "Codec used to decode stored values." }, "mode": { "description": "Redis connection mode.", "oneOf": [ { "type": "object", "title": "single", "properties": { "type": { "const": "single" }, "url": { "type": "string", "description": "Redis URL, e.g. redis://127.0.0.1:6379." } }, "required": [ "type", "url" ] }, { "type": "object", "title": "cluster", "properties": { "type": { "const": "cluster" }, "urls": { "type": "array", "items": { "type": "string" }, "description": "Redis cluster node URLs." } }, "required": [ "type", "urls" ] } ] }, "redis_type": { "description": "Redis structure holding the values.", "oneOf": [ { "type": "object", "title": "list", "properties": { "type": { "const": "list" } }, "required": [ "type" ] }, { "type": "object", "title": "string", "properties": { "type": { "const": "string" } }, "required": [ "type" ] } ] } }, "required": [ "type" ] } ] }, "job": { "type": "object", "description": "A local Job executed by the unified streaming kernel.", "additionalProperties": false, "required": [ "id", "version", "operators", "edges", "sources", "sinks" ], "properties": { "id": { "type": "string", "pattern": "^[A-Za-z0-9_-]+$", "description": "Job identity." }, "version": { "type": "integer", "minimum": 1, "description": "Monotonic Job version; upgrades restore compatible savepoints." }, "max_parallelism": { "type": "integer", "minimum": 1, "default": 1, "description": "Upper bound on physical task parallelism." }, "parallelism": { "type": "integer", "minimum": 1, "default": 1, "description": "Requested task parallelism." }, "operators": { "type": "array", "items": { "$ref": "#/$defs/job_operator" } }, "edges": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "id", "from", "to" ], "properties": { "id": { "type": "string" }, "from": { "type": "string" }, "to": { "type": "string" }, "partitioned": { "type": "boolean", "default": false, "description": "Route by key hash instead of broadcasting." } } } }, "sources": { "type": "array", "items": { "$ref": "#/$defs/job_source" } }, "sinks": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "operator_id", "output_type" ], "properties": { "operator_id": { "type": "string" }, "output_type": { "type": "string" }, "config": { "type": "object" } } } }, "state": { "type": "object", "additionalProperties": false, "required": [ "backend" ], "properties": { "backend": { "type": "string", "enum": [ "embedded_kv", "redb" ], "description": "Keyed-state backend (local execution)." }, "durability": { "type": "string", "enum": [ "durable", "ephemeral" ], "default": "durable", "description": "Whether state must survive process/node replacement." }, "root": { "type": "string", "description": "Stable working-state root; defaults to ARKFLOW_STATE_ROOT or data/arkflow-state." }, "namespace": { "type": "string" }, "ttl_ms": { "type": "integer", "minimum": 0 }, "format_version": { "type": "integer", "minimum": 1, "default": 1, "description": "State format contract; savepoint compatibility keys on it." }, "max_pending_transactions": { "type": "integer", "minimum": 1, "description": "Maximum simultaneously staged state-journal transactions (one per open window group or unacknowledged output). Raise it with realistic capacity planning when a window sees very high per-window key cardinality." }, "max_bytes": { "type": "integer", "minimum": 1, "description": "Optional live-byte budget for the disk-backed keyed-state backend." } } }, "checkpoint": { "type": "object", "additionalProperties": false, "properties": { "interval_ms": { "type": "integer", "minimum": 1 }, "object_store_uri": { "type": "string" }, "retention": { "type": "integer", "minimum": 1 } } }, "recovery": { "type": "string", "enum": [ "fail", "latest_checkpoint", "latest_savepoint" ], "default": "fail", "description": "Recovery policy on restart." } } }, "job_operator": { "type": "object", "additionalProperties": false, "required": [ "id", "kind" ], "properties": { "id": { "type": "string" }, "kind": { "type": "string", "enum": [ "source", "map", "aggregate", "window", "join", "sink" ] }, "stateful": { "type": "boolean", "default": false }, "key_field": { "type": "string" }, "config": { "type": "object" } } }, "job_source": { "type": "object", "additionalProperties": false, "required": [ "operator_id", "input_type", "time" ], "properties": { "operator_id": { "type": "string" }, "input_type": { "type": "string" }, "config": { "type": "object" }, "time": { "type": "object", "additionalProperties": false, "required": [ "mode" ], "properties": { "mode": { "type": "string", "enum": [ "processing_time", "event_time" ], "description": "Time semantics for this source." }, "timestamp_field": { "type": "string", "description": "Event-time field (Int64 ms or an Arrow timestamp column)." }, "watermark": { "type": "object", "additionalProperties": false, "properties": { "strategy": { "type": "string", "enum": [ "monotonous", "bounded_out_of_orderness" ] }, "out_of_orderness_ms": { "type": "integer", "minimum": 0 }, "idle_timeout_ms": { "type": "integer", "minimum": 0 } } }, "allowed_lateness_ms": { "type": "integer", "minimum": 0, "default": 0, "description": "Fired windows stay correctable (late Update) until end + lateness." }, "late_event_policy": { "type": "string", "enum": [ "drop", "update", "route" ], "default": "drop" }, "late_event_route": { "type": "string", "description": "Operator receiving routed late/invalid-timestamp rows." } } } } }, "stream": { "type": "object", "additionalProperties": false, "required": [ "input", "output", "pipeline" ], "properties": { "id": { "type": "string", "pattern": "^[A-Za-z0-9_-]+$", "description": "Stable logical identifier for control-plane operations." }, "input": { "$ref": "#/$defs/input" }, "output": { "$ref": "#/$defs/output" }, "error_output": { "$ref": "#/$defs/output" }, "pipeline": { "type": "object", "additionalProperties": false, "required": [ "processors" ], "properties": { "thread_num": { "type": "integer", "minimum": 1, "description": "Number of processor worker threads. Defaults to the number of CPU cores when omitted." }, "processors": { "type": "array", "items": { "$ref": "#/$defs/processor" } }, "input": { "type": "object", "description": "Stream-level input overrides (currently unused, reserved for future).", "additionalProperties": true }, "output": { "type": "object", "description": "Stream-level output overrides (currently unused, reserved for future).", "additionalProperties": true } } }, "buffer": { "$ref": "#/$defs/buffer" }, "state": { "type": "object", "additionalProperties": false, "required": [ "backend" ], "properties": { "backend": { "type": "string", "enum": [ "embedded_kv", "redb" ], "description": "Keyed-state backend for legacy stream windows." }, "durability": { "type": "string", "enum": [ "durable", "ephemeral" ], "default": "durable", "description": "Whether state must survive process/node replacement." }, "root": { "type": "string", "description": "Stable working-state root; durable state also requires a checkpoint, so use a Job for recovery." }, "namespace": { "type": "string" }, "ttl_ms": { "type": "integer", "minimum": 0 }, "format_version": { "type": "integer", "minimum": 1, "default": 1 }, "max_pending_transactions": { "type": "integer", "minimum": 1 }, "max_bytes": { "type": "integer", "minimum": 1 } } }, "temporary": { "type": "array", "description": "Optional temporary (lookup) components shared by the pipeline.", "items": { "$ref": "#/$defs/temporary" } } } } } }