{ "version": 2, "components": [ { "kind": "input", "name": "file", "description": "Reads records from local or remote object storage (S3, GCS, Azure, HDFS) in CSV/JSON/Parquet/Avro/Arrow formats.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "input_type": { "type": "object", "description": "Format-specific input settings (type: csv/json/parquet/avro/arrow)." }, "ballista": { "type": "object", "description": "Optional Ballista distributed compute configuration." }, "query": { "type": "object", "description": "Optional SQL query and table name to filter the read." } }, "required": [ "input_type" ] }, "config_example": { "input_type": { "type": "parquet", "path": "s3://bucket/data.parquet" } } }, { "kind": "input", "name": "generate", "description": "Generates synthetic text messages on a fixed interval (useful for testing and load simulation).", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "context": { "type": "string", "description": "Payload string emitted on every read." }, "interval": { "type": "string", "description": "Delay between batches (e.g. '100ms', '1s'). Accepts any humantime duration." }, "count": { "type": "integer", "minimum": 1, "description": "Optional total number of messages to emit before signalling EOF." }, "batch_size": { "type": "integer", "minimum": 1, "default": 1, "description": "Number of messages returned per read call." } }, "required": [ "context", "interval" ] }, "config_example": { "context": "hello", "interval": "1s", "count": 100, "batch_size": 10 } }, { "kind": "input", "name": "http", "description": "Receives data via HTTP. Can run as a server (POST/PUT on `path`) or poll a remote endpoint.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "address": { "type": "string", "description": "Bind address (server mode) or base URL (client mode)." }, "path": { "type": "string", "description": "Path to accept messages on (server mode) or full request path (client mode)." }, "method": { "type": "string", "enum": [ "GET", "POST", "PUT", "DELETE", "PATCH" ], "description": "HTTP method (client mode)." }, "interval": { "type": "string", "description": "Polling interval in humantime format (client mode, e.g. '5s')." }, "cors_enabled": { "type": "boolean", "default": false, "description": "Enable CORS for the server." }, "auth": { "type": "object", "description": "Authentication configuration." } }, "required": [ "address", "path" ] }, "config_example": { "address": "0.0.0.0:8080", "path": "/ingest", "cors_enabled": true } }, { "kind": "input", "name": "kafka", "description": "Consumes messages from Apache Kafka topics with a consumer group.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "brokers": { "type": "array", "items": { "type": "string" }, "description": "List of Kafka broker addresses." }, "topics": { "type": "array", "items": { "type": "string" }, "description": "Topics to subscribe to." }, "consumer_group": { "type": "string", "description": "Consumer group ID for offset coordination." }, "client_id": { "type": "string", "description": "Optional client identifier." }, "start_from_latest": { "type": "boolean", "default": false, "description": "When true, ignore committed offsets and start from the latest message." }, "fetch_min_bytes": { "type": "integer", "minimum": 0, "description": "Minimum bytes before the broker responds to a fetch request." }, "fetch_max_bytes": { "type": "integer", "minimum": 0, "description": "Maximum bytes for a fetch request." }, "fetch_max_partition_bytes": { "type": "integer", "minimum": 0, "description": "Maximum bytes per partition in a fetch request." }, "fetch_wait_max_ms": { "type": "integer", "minimum": 0, "description": "Maximum time to wait for fetch data in milliseconds." }, "batch_max_rows": { "type": "integer", "minimum": 1, "default": 1024, "description": "Maximum number of messages aggregated into one read batch (clamped to 1 minimum; 1 restores per-message batches)." }, "batch_max_bytes": { "type": "integer", "minimum": 1, "default": 8388608, "description": "Maximum accumulated payload bytes per read batch; the first message is always included (clamped to 1 minimum)." }, "security": { "type": "object", "description": "SASL authentication and TLS settings. Omit entirely for plaintext.", "properties": { "protocol": { "type": "string", "enum": [ "plaintext", "ssl", "sasl_plaintext", "sasl_ssl" ], "description": "Security protocol. Defaults to inferred from which sub-blocks are present: both -> sasl_ssl, only sasl -> sasl_plaintext, only tls -> ssl, neither -> plaintext." }, "sasl": { "type": "object", "description": "SASL credentials (PLAIN or SCRAM).", "properties": { "mechanism": { "type": "string", "enum": [ "plain", "scram-sha-256", "scram-sha-512" ], "description": "SASL mechanism." }, "username": { "type": "string", "description": "SASL username (required for plain/scram)." }, "password": { "type": "string", "description": "SASL password (required for plain/scram)." } }, "required": [ "mechanism" ] }, "tls": { "type": "object", "description": "TLS settings. ca/cert/key accept a file path or inline PEM text (auto-detected).", "properties": { "ca": { "type": "string", "description": "CA certificate for verifying the broker: file path or inline PEM." }, "cert": { "type": "string", "description": "Client certificate for mTLS: file path or inline PEM." }, "key": { "type": "string", "description": "Client private key for mTLS: file path or inline PEM." }, "key_password": { "type": "string", "description": "Password protecting the client private key." }, "insecure_skip_verify": { "type": "boolean", "default": false, "description": "Disable broker certificate verification. Development/testing only." } } } } } }, "required": [ "brokers", "topics", "consumer_group" ] }, "config_example": { "brokers": [ "localhost:9092" ], "topics": [ "events" ], "consumer_group": "arkflow", "start_from_latest": false } }, { "kind": "input", "name": "memory", "description": "In-memory input queue seeded with an initial list of messages. Primarily for tests and demos.", "config_optional": true, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "messages": { "type": "array", "items": { "type": "string" }, "description": "Initial messages to enqueue." } } }, "config_example": { "messages": [ "hello", "world" ] } }, { "kind": "input", "name": "modbus", "description": "Polls Modbus TCP devices on a fixed interval, reading coils, discrete inputs, or registers.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "addr": { "type": "string", "description": "Modbus server address (host:port)." }, "slave_id": { "type": "integer", "description": "Modbus unit / slave ID." }, "points": { "type": "array", "description": "Points to read on every poll.", "items": { "type": "object", "properties": { "type": { "type": "string", "enum": [ "coils", "discrete_inputs", "holding_registers", "input_registers" ] }, "name": { "type": "string" }, "address": { "type": "integer" }, "quantity": { "type": "integer", "minimum": 1 } }, "required": [ "type", "name", "address", "quantity" ] } }, "interval": { "type": "string", "description": "Poll interval (humantime)." } }, "required": [ "addr", "slave_id", "points", "interval" ] }, "config_example": { "addr": "127.0.0.1:502", "slave_id": 1, "interval": "1s", "points": [ { "type": "holding_registers", "name": "voltage", "address": 0, "quantity": 1 } ] } }, { "kind": "input", "name": "mqtt", "description": "Subscribes to an MQTT broker and forwards messages from the configured topics.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "host": { "type": "string", "description": "MQTT broker hostname." }, "port": { "type": "integer", "minimum": 1, "maximum": 65535, "description": "MQTT broker port." }, "client_id": { "type": "string", "description": "Unique client identifier." }, "username": { "type": "string", "description": "Optional username." }, "password": { "type": "string", "description": "Optional password." }, "topics": { "type": "array", "items": { "type": "string" }, "description": "Topics to subscribe to (MQTT wildcards supported)." }, "qos": { "type": "integer", "enum": [ 0, 1, 2 ], "default": 0, "description": "Quality of Service level." }, "clean_session": { "type": "boolean", "default": true, "description": "Whether to use a clean session." }, "keep_alive": { "type": "integer", "minimum": 1, "description": "Keep-alive interval in seconds." }, "tls": { "type": "object", "description": "TLS transport configuration.", "properties": { "enabled": { "type": "boolean", "default": true }, "ca": { "type": "string", "description": "CA certificate file path." }, "client_cert": { "type": "string", "description": "Client certificate file (mTLS)." }, "client_key": { "type": "string", "description": "Client private key file (mTLS)." } } } }, "required": [ "host", "port", "client_id", "topics" ] }, "config_example": { "host": "localhost", "port": 1883, "client_id": "arkflow", "topics": [ "sensors/#" ] } }, { "kind": "input", "name": "multiple_inputs", "description": "Combines multiple input sources into a single stream. Each source is tagged with __meta_source.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "inputs": { "type": "array", "minItems": 1, "description": "List of input components to combine.", "items": { "type": "object", "properties": { "type": { "type": "string", "description": "Input type (e.g. 'kafka', 'http')." }, "name": { "type": "string", "description": "Optional logical name for this source." } }, "required": [ "type" ] } } }, "required": [ "inputs" ] }, "config_example": { "inputs": [ { "type": "kafka", "name": "events", "brokers": [ "localhost:9092" ], "topics": [ "events" ], "consumer_group": "arkflow", "start_from_latest": false }, { "type": "kafka", "name": "logs", "brokers": [ "localhost:9092" ], "topics": [ "logs" ], "consumer_group": "arkflow", "start_from_latest": false } ] } }, { "kind": "input", "name": "nats", "description": "Consumes messages from NATS, supporting both regular subjects and JetStream consumers.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "NATS server URL (e.g. nats://localhost:4222)." }, "mode": { "type": "object", "description": "Select between plain NATS and JetStream subscriptions.", "oneOf": [ { "properties": { "type": { "const": "regular" }, "subject": { "type": "string" }, "queue_group": { "type": "string" } }, "required": [ "type", "subject" ] }, { "properties": { "type": { "const": "jet_stream" }, "stream": { "type": "string" }, "consumer_name": { "type": "string" }, "durable_name": { "type": "string" } }, "required": [ "type", "stream", "consumer_name" ] } ] }, "auth": { "type": "object", "description": "NATS authentication configuration." } }, "required": [ "url", "mode" ] }, "config_example": { "url": "nats://localhost:4222", "mode": { "type": "regular", "subject": "events" } } }, { "kind": "input", "name": "pulsar", "description": "Subscribes to an Apache Pulsar topic with configurable subscription type and authentication.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "service_url": { "type": "string", "description": "Pulsar service URL (e.g. pulsar://localhost:6650)." }, "topic": { "type": "string", "description": "Topic to subscribe to." }, "subscription_name": { "type": "string", "description": "Subscription name." }, "subscription_type": { "type": "string", "enum": [ "Exclusive", "Shared", "Failover", "KeyShared" ], "description": "Subscription type." }, "auth": { "type": "object", "description": "Pulsar authentication configuration." }, "retry_config": { "type": "object", "description": "Retry behaviour for failed messages." } }, "required": [ "service_url", "topic", "subscription_name" ] }, "config_example": { "service_url": "pulsar://localhost:6650", "topic": "persistent://public/default/events", "subscription_name": "arkflow" } }, { "kind": "input", "name": "redis", "description": "Reads from Redis: list blocking pops, pub/sub subscriptions, or stream consumer groups.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "mode": { "type": "object", "description": "Connection mode.", "oneOf": [ { "properties": { "type": { "const": "cluster" }, "urls": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "urls" ] }, { "properties": { "type": { "const": "single" }, "url": { "type": "string" } }, "required": [ "type", "url" ] } ] }, "redis_type": { "type": "object", "description": "Data structure to consume from.", "oneOf": [ { "properties": { "type": { "const": "subscribe" }, "subscribe": { "type": "object", "oneOf": [ { "properties": { "type": { "const": "channels" }, "channels": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "channels" ] }, { "properties": { "type": { "const": "patterns" }, "patterns": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "patterns" ] } ] } }, "required": [ "type", "subscribe" ] }, { "properties": { "type": { "const": "list" }, "list": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "list" ] } ] } }, "required": [ "mode", "redis_type" ] }, "config_example": { "mode": { "type": "single", "url": "redis://localhost:6379" }, "redis_type": { "type": "list", "list": [ "events" ] } } }, { "kind": "input", "name": "sql", "description": "Runs a one-shot SELECT against a SQL database (MySQL / PostgreSQL / SQLite / DuckDB) and emits the rows as batches.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "select_sql": { "type": "string", "description": "SELECT statement executed once at startup." }, "ballista": { "type": "object", "description": "Optional Ballista distributed compute configuration." }, "input_type": { "type": "object", "description": "Database connection settings." } }, "required": [ "select_sql", "input_type" ] }, "config_example": { "select_sql": "SELECT id, name FROM users", "input_type": { "type": "sqlite", "name": "users", "path": "./data.db" } } }, { "kind": "input", "name": "websocket", "description": "Connects to a WebSocket server and forwards each incoming message as a batch.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "WebSocket server URL (ws:// or wss://)." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Headers included in the WebSocket handshake." }, "timeout": { "type": "integer", "minimum": 1, "description": "Connection timeout in seconds." } }, "required": [ "url" ] }, "config_example": { "url": "ws://localhost:8080/stream" } }, { "kind": "output", "name": "drop", "description": "Discards all messages. Useful for performance benchmarks and dead-end pipelines.", "config_optional": true, "config_schema": { "type": "object" } }, { "kind": "output", "name": "http", "description": "Posts each batch to an HTTP endpoint. Supports custom headers, retry, and auth.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "Destination URL." }, "method": { "type": "string", "enum": [ "GET", "POST", "PUT", "DELETE", "PATCH" ], "default": "POST", "description": "HTTP method." }, "timeout_ms": { "type": "integer", "minimum": 1, "description": "Request timeout in milliseconds." }, "retry_count": { "type": "integer", "minimum": 0, "description": "Number of retry attempts on failure." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Custom HTTP headers." }, "body_field": { "type": "string", "description": "Record field that holds the request body." }, "auth": { "type": "object", "description": "Authentication configuration." } }, "required": [ "url" ] }, "config_example": { "url": "https://example.com/ingest", "method": "POST", "timeout_ms": 5000, "retry_count": 3 } }, { "kind": "output", "name": "influxdb", "description": "Writes time-series data to InfluxDB v2.x using the Line Protocol.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "InfluxDB server URL (e.g. http://localhost:8086)." }, "org": { "type": "string", "description": "Organization name." }, "bucket": { "type": "string", "description": "Destination bucket." }, "token": { "type": "string", "description": "Authentication token." }, "measurement": { "type": "string", "description": "Measurement name." }, "tags": { "type": "array", "items": { "type": "object", "properties": { "field": { "type": "string" }, "tag_name": { "type": "string" } }, "required": [ "field", "tag_name" ] }, "description": "Tag mappings (label fields)." }, "fields": { "type": "array", "items": { "type": "object", "properties": { "field": { "type": "string" }, "field_name": { "type": "string" }, "field_type": { "enum": [ "float", "integer", "boolean", "string" ] } }, "required": [ "field", "field_name" ] }, "description": "Field mappings (value fields)." }, "timestamp_field": { "type": "string", "description": "Source field for the point timestamp." }, "batch_size": { "type": "integer", "minimum": 1, "description": "Batch size for write requests." }, "flush_interval": { "type": "integer", "minimum": 1, "description": "Maximum seconds to wait before flushing a partial batch." }, "retry_count": { "type": "integer", "minimum": 1, "description": "Number of HTTP attempts per flush." }, "timeout_ms": { "type": "integer", "minimum": 1, "description": "HTTP request timeout in milliseconds." } }, "required": [ "url", "org", "bucket", "token", "measurement", "fields" ] }, "config_example": { "url": "http://localhost:8086", "org": "arkflow", "bucket": "metrics", "token": "${INFLUX_TOKEN}", "measurement": "sensor", "tags": [ { "field": "sensor", "tag_name": "sensor" } ], "fields": [ { "field": "value", "field_name": "value", "field_type": "float" } ], "timestamp_field": "timestamp" } }, { "kind": "output", "name": "kafka", "description": "Produces messages to Apache Kafka. Supports key-based partitioning and compression.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "brokers": { "type": "array", "items": { "type": "string" }, "description": "List of Kafka broker addresses." }, "topic": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ], "description": "Literal value or a SQL expression evaluated per batch." }, "key": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ], "description": "Literal value or a SQL expression evaluated per batch." }, "client_id": { "type": "string", "description": "Optional client identifier." }, "compression": { "type": "string", "enum": [ "none", "gzip", "snappy", "lz4", "zstd" ], "description": "Compression algorithm." }, "acks": { "type": "string", "enum": [ "0", "1", "all" ], "description": "Acknowledgment level." }, "value_field": { "type": "string", "description": "Record field used as the message payload." }, "exactly_once": { "type": "boolean", "default": false, "description": "Enable exactly-once transactional production (L2)." }, "transactional_id": { "type": "string", "description": "Transactional id (required when exactly_once is true); must be stable across restarts for zombie fencing." }, "offset_commit_group": { "type": "string", "description": "L3 exactly-once: consumer group of a paired Kafka input (with transactional_offsets: true) whose source offsets commit inside this output's producer transactions. Requires exactly_once." }, "security": { "type": "object", "description": "SASL authentication and TLS settings. Omit entirely for plaintext.", "properties": { "protocol": { "type": "string", "enum": [ "plaintext", "ssl", "sasl_plaintext", "sasl_ssl" ], "description": "Security protocol. Defaults to inferred from which sub-blocks are present: both -> sasl_ssl, only sasl -> sasl_plaintext, only tls -> ssl, neither -> plaintext." }, "sasl": { "type": "object", "description": "SASL credentials (PLAIN or SCRAM).", "properties": { "mechanism": { "type": "string", "enum": [ "plain", "scram-sha-256", "scram-sha-512" ], "description": "SASL mechanism." }, "username": { "type": "string", "description": "SASL username (required for plain/scram)." }, "password": { "type": "string", "description": "SASL password (required for plain/scram)." } }, "required": [ "mechanism" ] }, "tls": { "type": "object", "description": "TLS settings. ca/cert/key accept a file path or inline PEM text (auto-detected).", "properties": { "ca": { "type": "string", "description": "CA certificate for verifying the broker: file path or inline PEM." }, "cert": { "type": "string", "description": "Client certificate for mTLS: file path or inline PEM." }, "key": { "type": "string", "description": "Client private key for mTLS: file path or inline PEM." }, "key_password": { "type": "string", "description": "Password protecting the client private key." }, "insecure_skip_verify": { "type": "boolean", "default": false, "description": "Disable broker certificate verification. Development/testing only." } } } } } }, "required": [ "brokers", "topic" ] }, "config_example": { "brokers": [ "localhost:9092" ], "topic": { "type": "value", "value": "events" } } }, { "kind": "output", "name": "milvus", "description": "Upserts batch rows into a Milvus collection over the REST v2 vectordb API: a Float32 list column becomes the vector, other columns pack into a JSON payload field, and an optional id column keys the row.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "Milvus base URL, e.g. http://localhost:19530." }, "collection": { "type": "string", "description": "Target collection name (schema must already exist)." }, "vector_field": { "type": "string", "description": "Name of the vector field (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "id_field": { "type": "string", "description": "Field carrying the row id (integer or string). Required: the upsert API needs a primary key per row (auto-id collections are not supported)." }, "payload_field": { "type": "string", "description": "JSON field receiving every remaining column as a per-row object. Defaults to 'payload'; set to an empty string to disable." }, "api_key": { "type": "string", "description": "Sent as 'Authorization: Bearer' (Milvus convention: :); supports secret references." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "retry_count": { "type": "integer", "description": "Retry attempts for connection errors and 5xx responses. Defaults to 0." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." } }, "required": [ "url", "collection", "id_field" ] }, "config_example": { "url": "http://localhost:19530", "collection": "documents", "vector_field": "embedding", "id_field": "doc_id", "api_key": "${env:MILVUS_CREDENTIALS}" } }, { "kind": "output", "name": "mongodb", "description": "Writes Arrow rows to MongoDB as BSON documents.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "uri": { "type": "string", "description": "MongoDB connection URI." }, "database": { "type": "string", "description": "Target database name." }, "collection": { "type": "string", "description": "Target collection name." } }, "required": [ "uri", "database", "collection" ] }, "config_example": { "uri": "mongodb://localhost:27017", "database": "arkflow", "collection": "events" } }, { "kind": "output", "name": "mqtt", "description": "Publishes messages to an MQTT broker topic.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "host": { "type": "string", "description": "MQTT broker hostname." }, "port": { "type": "integer", "minimum": 1, "maximum": 65535, "description": "MQTT broker port." }, "client_id": { "type": "string", "description": "Client identifier." }, "username": { "type": "string", "description": "Optional username." }, "password": { "type": "string", "description": "Optional password." }, "topic": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ], "description": "Literal value or a SQL expression evaluated per batch." }, "qos": { "type": "integer", "enum": [ 0, 1, 2 ], "default": 0, "description": "Quality of Service." }, "clean_session": { "type": "boolean", "default": true }, "keep_alive": { "type": "integer", "minimum": 1, "description": "Keep-alive interval in seconds." }, "retain": { "type": "boolean", "default": false, "description": "Whether to retain the message on the broker." }, "value_field": { "type": "string", "description": "Column whose per-row value becomes the message payload (binary or string column). When unset, the codec encoding of the batch applies." }, "tls": { "type": "object", "description": "TLS transport configuration.", "properties": { "enabled": { "type": "boolean", "default": true }, "ca": { "type": "string", "description": "CA certificate file path." }, "client_cert": { "type": "string", "description": "Client certificate file (mTLS)." }, "client_key": { "type": "string", "description": "Client private key file (mTLS)." } } } }, "required": [ "host", "port", "client_id", "topic" ] }, "config_example": { "host": "localhost", "port": 1883, "client_id": "arkflow", "topic": { "type": "value", "value": "sensors/data" } } }, { "kind": "output", "name": "nats", "description": "Publishes to NATS, either to a regular subject or a JetStream stream.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "NATS server URL." }, "mode": { "type": "object", "description": "Select regular or JetStream publishing.", "oneOf": [ { "properties": { "type": { "const": "regular" }, "subject": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "subject" ] }, { "properties": { "type": { "const": "jet_stream" }, "stream": { "type": "string" }, "subject": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "stream", "subject" ] } ] }, "auth": { "type": "object", "description": "NATS authentication configuration." }, "value_field": { "type": "string", "description": "Record field used as the payload." } }, "required": [ "url", "mode" ] }, "config_example": { "url": "nats://localhost:4222", "mode": { "type": "regular", "subject": { "type": "value", "value": "events" } } } }, { "kind": "output", "name": "pgvector", "description": "Upserts batch rows into a PostgreSQL table with the pgvector extension: a Float32 list column becomes the vector, other columns are packed into a jsonb payload, and an optional id column keys ON CONFLICT upserts.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "Postgres connection string, e.g. postgres://user:pass@host:5432/db; supports secret references." }, "table": { "type": "string", "description": "Target table (must already exist with a matching vector column)." }, "vector_field": { "type": "string", "description": "Name of the vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "id_field": { "type": "string", "description": "Column used as the upsert conflict key (integer or string). When omitted, plain INSERTs are written." }, "payload_field": { "type": "string", "description": "jsonb column receiving every remaining column as a per-row JSON object. Defaults to 'payload'; set to an empty string to disable." }, "max_connections": { "type": "integer", "description": "Connection pool size. Defaults to 4." }, "timeout_ms": { "type": "integer", "description": "Pool acquire/connect timeout in milliseconds. Defaults to 30000." } }, "required": [ "url", "table" ] }, "config_example": { "url": "postgres://postgres:postgres@localhost:5432/vectors", "table": "documents", "vector_field": "embedding", "id_field": "doc_id" } }, { "kind": "output", "name": "pulsar", "description": "Produces messages to an Apache Pulsar topic.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "service_url": { "type": "string", "description": "Pulsar service URL." }, "topic": { "type": "string", "description": "Destination topic (supports {field} placeholders)." }, "auth": { "type": "object", "description": "Pulsar authentication configuration." }, "value_field": { "type": "string", "description": "Record field used as the payload." } }, "required": [ "service_url", "topic" ] }, "config_example": { "service_url": "pulsar://localhost:6650", "topic": "persistent://public/default/events" } }, { "kind": "output", "name": "qdrant", "description": "Upserts batch rows as Qdrant points over the REST API: a vector column, an optional id column, and all remaining columns as payload.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "Qdrant base URL, e.g. http://localhost:6333." }, "collection": { "type": "string", "description": "Target collection name." }, "vector_field": { "type": "string", "description": "Name of the vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "id_field": { "type": "string", "description": "Column used as the point id (unsigned integer or string). When omitted, Qdrant generates ids." }, "payload_fields": { "type": "array", "items": { "type": "string" }, "description": "Columns to include in the point payload. Defaults to every column except the vector and id columns." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer'; supports secret references." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "retry_count": { "type": "integer", "description": "Retry attempts for connection errors and 5xx responses. Defaults to 3." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." } }, "required": [ "url", "collection" ] }, "config_example": { "url": "http://localhost:6333", "collection": "documents", "vector_field": "embedding", "id_field": "doc_id" } }, { "kind": "output", "name": "redis", "description": "Writes messages to Redis via publish, list rpush, hashes, or plain strings.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "mode": { "type": "object", "description": "Connection mode (single or cluster).", "oneOf": [ { "properties": { "type": { "const": "cluster" }, "urls": { "type": "array", "items": { "type": "string" } } }, "required": [ "type", "urls" ] }, { "properties": { "type": { "const": "single" }, "url": { "type": "string" } }, "required": [ "type", "url" ] } ] }, "redis_type": { "type": "object", "description": "Destination data structure.", "oneOf": [ { "properties": { "type": { "const": "publish" }, "channel": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "channel" ] }, { "properties": { "type": { "const": "list" }, "key": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "key" ] }, { "properties": { "type": { "const": "hashes" }, "key": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] }, "field": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "key", "field" ] }, { "properties": { "type": { "const": "strings" }, "key": { "oneOf": [ { "type": "object", "properties": { "type": { "const": "value" }, "value": { "type": "string" } }, "required": [ "type", "value" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "const": "expr" }, "expr": { "type": "string" } }, "required": [ "type", "expr" ], "additionalProperties": false } ] } }, "required": [ "type", "key" ] } ] }, "value_field": { "type": "string", "description": "Record field used as the payload." } }, "required": [ "mode", "redis_type" ] }, "config_example": { "mode": { "type": "single", "url": "redis://localhost:6379" }, "redis_type": { "type": "publish", "channel": { "type": "value", "value": "events" } } } }, { "kind": "output", "name": "sql", "description": "Batch-inserts records into a MySQL or PostgreSQL database, with optional upsert (ON DUPLICATE KEY UPDATE / ON CONFLICT DO UPDATE) for idempotent writes.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "output_type": { "type": "object", "description": "Database connection settings, tagged by `type` (`mysql` or `postgres`), each with `uri` and optional `ssl`.", "properties": { "type": { "type": "string", "enum": [ "mysql", "postgres" ] }, "uri": { "type": "string" }, "ssl": { "type": "object", "description": "Optional SSL configuration (ssl_mode, root_cert, client_cert, client_key)." } }, "required": [ "type", "uri" ] }, "table_name": { "type": "string", "description": "Destination table." }, "upsert": { "type": "boolean", "default": false, "description": "Use upsert (MySQL ON DUPLICATE KEY UPDATE / PostgreSQL ON CONFLICT DO UPDATE) instead of a plain insert." }, "upsert_keys": { "type": "array", "items": { "type": "string" }, "description": "Columns used as the conflict target for upsert. Required (non-empty) when upsert is true." } }, "required": [ "output_type", "table_name" ] }, "config_example": { "output_type": { "type": "postgres", "uri": "postgres://user:pass@localhost/db" }, "table_name": "events", "upsert": true, "upsert_keys": [ "id" ] } }, { "kind": "output", "name": "stdout", "description": "Writes each message to the console. Useful for debugging and demos.", "config_optional": true, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "append_newline": { "type": "boolean", "default": true, "description": "Append a newline after each message." } } }, "config_example": { "append_newline": true } }, { "kind": "processor", "name": "arrow_to_json", "description": "Converts an Arrow RecordBatch into JSON byte payloads (one per row).", "config_optional": true, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "value_field": { "type": "string", "description": "Binary column carrying one JSON payload per row (defaults to the standard binary value column)." }, "fields_to_include": { "type": "array", "items": { "type": "string" }, "description": "Only emit these top-level fields." } } } }, { "kind": "processor", "name": "arrow_to_protobuf", "description": "Serializes Arrow RecordBatches into Protobuf wire-format bytes.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "message_type": { "type": "string", "description": "Fully-qualified Protobuf message type name." }, "proto_inputs": { "type": "array", "items": { "type": "string" }, "description": "Paths to .proto files." }, "proto_includes": { "type": "array", "items": { "type": "string" }, "description": "Include paths for proto resolution." }, "fields_to_include": { "type": "array", "items": { "type": "string" }, "description": "Optional allow-list of field names to include when serializing to Protobuf." } }, "required": [ "message_type", "proto_inputs" ] } }, { "kind": "processor", "name": "batch", "description": "Batches messages by row count with an idle timeout before forwarding.", "config_optional": true, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "count": { "type": "integer", "minimum": 1, "description": "Number of message rows to accumulate before flushing." }, "timeout_ms": { "type": "integer", "minimum": 1, "description": "Idle timeout that flushes a partial batch (milliseconds)." } }, "required": [ "count", "timeout_ms" ] }, "config_example": { "count": 100, "timeout_ms": 5000 } }, { "kind": "processor", "name": "embedding", "description": "Batch-embeds a text column through an OpenAI-compatible embeddings API and appends the vectors as a FixedSizeList(Float32) column.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "api_base": { "type": "string", "description": "Base URL of the embeddings API, e.g. https://api.openai.com/v1 (the request path is {api_base}/embeddings)." }, "model": { "type": "string", "description": "Embedding model name, e.g. text-embedding-3-small." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer'; supports secret references. Omit for unauthenticated endpoints." }, "field": { "type": "string", "description": "Name of the input UTF-8 column to embed." }, "target_field": { "type": "string", "description": "Name of the appended vector column. Defaults to 'embedding'." }, "batch_size": { "type": "integer", "description": "Maximum rows per HTTP request. Defaults to 32." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers, e.g. Azure's 'api-key'." } }, "required": [ "api_base", "model", "field" ] }, "config_example": { "api_base": "https://api.openai.com/v1", "model": "text-embedding-3-small", "api_key": "${env:OPENAI_API_KEY}", "field": "text", "batch_size": 32 } }, { "kind": "processor", "name": "json_to_arrow", "description": "Parses JSON byte payloads into an Arrow RecordBatch with inferred schema.", "config_optional": true, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "value_field": { "type": "string", "description": "Binary column carrying one JSON payload per row (defaults to the standard binary value column)." }, "fields_to_include": { "type": "array", "items": { "type": "string" }, "description": "Only parse these top-level fields into columns." } } } }, { "kind": "processor", "name": "llm", "description": "Sends each row of a text column to an OpenAI-compatible chat completions API and appends the completion as a Utf8 column, with bounded ordered concurrency.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "api_base": { "type": "string", "description": "Base URL of the API, e.g. https://api.openai.com/v1 (the request path is {api_base}/chat/completions)." }, "model": { "type": "string", "description": "Chat model name, e.g. gpt-4o-mini." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer'; supports secret references. Omit for unauthenticated endpoints." }, "field": { "type": "string", "description": "Name of the input UTF-8 column to send." }, "target_field": { "type": "string", "description": "Name of the appended completion column. Defaults to 'response'." }, "system_prompt": { "type": "string", "description": "Optional system message prepended to each request." }, "prompt_template": { "type": "string", "description": "Optional user-message template; '{{value}}' is replaced with the row text. When omitted the row text is the user message." }, "temperature": { "type": "number", "description": "Sampling temperature; omitted from the request when unset." }, "max_tokens": { "type": "integer", "description": "Completion token cap; omitted from the request when unset." }, "concurrency": { "type": "integer", "description": "Maximum in-flight requests. Defaults to 4." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." }, "stream": { "type": "boolean", "description": "Request SSE streaming (`stream: true`) and assemble the deltas into the row's final completion text. Useful behind providers or gateways that time out non-streaming long completions." }, "tools": { "description": "OpenAI tools definition passed through verbatim to the request body for function calling." }, "tool_calls_column": { "type": "string", "description": "When set, appends this column with the response's tool_calls as JSON text (empty string when the response has none). Streamed tool_call deltas are merged by index." } }, "required": [ "api_base", "model", "field" ] }, "config_example": { "api_base": "https://api.openai.com/v1", "model": "gpt-4o-mini", "api_key": "${env:OPENAI_API_KEY}", "field": "text", "system_prompt": "You classify support tickets with one word.", "concurrency": 4 } }, { "kind": "processor", "name": "lookup", "description": "Resolves each row's lookup key against an external dimension service and appends the configured result columns, with an in-process TTL cache, negative caching, and bounded ordered concurrency.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "key_columns": { "type": "array", "items": { "type": "string" }, "minItems": 1, "description": "Input column(s) forming the lookup key (Utf8/Int64/UInt64/Boolean). Multiple columns join into a composite key in configured order; a null key cell follows the missing-key semantics." }, "columns": { "type": "array", "items": { "type": "object", "additionalProperties": false, "properties": { "name": { "type": "string", "description": "Appended output column name." }, "path": { "type": "string", "description": "Dot path of the value inside the backend's JSON record, e.g. 'profile.age'." }, "type": { "type": "string", "enum": [ "utf8", "int64", "float64", "bool" ], "description": "Output column type. A value that is absent or not coercible yields a null cell. Defaults to 'utf8'." } }, "required": [ "name", "path" ] }, "minItems": 1, "description": "Result columns appended to the batch. Names must not collide with existing input columns." }, "backend": { "type": "object", "description": "Dimension service backend. Currently only 'http' is available.", "oneOf": [ { "type": "object", "additionalProperties": false, "properties": { "http": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "URL template containing '{{key}}', substituted per lookup key." }, "method": { "type": "string", "description": "HTTP method. Defaults to 'GET'." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." }, "encode_key": { "type": "boolean", "description": "Percent-encode the substituted key (RFC 3986 unreserved set kept literal). Defaults to true." } }, "required": [ "url" ] } }, "required": [ "http" ] } ] }, "ttl_ms": { "type": "integer", "description": "Cache TTL for resolved records in milliseconds. Defaults to 60000." }, "negative_ttl_ms": { "type": "integer", "description": "Cache TTL for misses (negative caching) in milliseconds. Defaults to ttl_ms." }, "max_entries": { "type": "integer", "description": "Cache capacity bound; expired entries are swept first, then the least recently used are evicted. Defaults to 100000." }, "concurrency": { "type": "integer", "description": "Maximum in-flight backend requests. Defaults to 8." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "on_missing": { "type": "string", "enum": [ "null", "discard" ], "description": "Unresolved key handling: keep the row with null lookup columns (default) or drop the row." }, "on_backend_error": { "type": "string", "enum": [ "fail", "null" ], "description": "Backend failure handling: fail the batch (default) or degrade the affected keys to uncached misses." } }, "required": [ "key_columns", "columns", "backend" ] }, "config_example": { "key_columns": [ "user_id" ], "columns": [ { "name": "user_name", "path": "profile.name", "type": "utf8" }, { "name": "tier", "path": "profile.tier", "type": "utf8" } ], "backend": { "http": { "url": "http://dim-service/users/{{key}}" } }, "ttl_ms": 60000 } }, { "kind": "processor", "name": "milvus_search", "description": "Searches a Milvus collection for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column. Rows are searched with bounded, order-preserving concurrency.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "Milvus base URL, e.g. http://localhost:19530." }, "collection": { "type": "string", "description": "Collection to search." }, "vector_field": { "type": "string", "description": "Name of the query vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "target_field": { "type": "string", "description": "Name of the appended matches column (JSON array text). Defaults to 'matches'." }, "id_field": { "type": "string", "description": "Collection field returned as the match id. When omitted the id key is left out (auto-id collections)." }, "payload_field": { "type": "string", "description": "Collection field included in each match as a payload object. Defaults to 'payload'; set to an empty string to disable." }, "metric": { "type": "string", "enum": [ "COSINE", "L2", "IP" ], "description": "Metric type; must match the collection schema. Defaults to COSINE." }, "top_k": { "type": "integer", "description": "Number of neighbors per row. Defaults to 5." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer' (Milvus convention: :); supports secret references." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." } }, "required": [ "url", "collection" ] }, "config_example": { "url": "http://localhost:19530", "collection": "documents", "vector_field": "embedding", "id_field": "doc_id", "top_k": 5 } }, { "kind": "processor", "name": "pgvector_search", "description": "Searches a PostgreSQL table with the pgvector extension for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "Postgres connection string; supports secret references." }, "table": { "type": "string", "description": "Table to search (must already exist with a pgvector column)." }, "vector_field": { "type": "string", "description": "Name of the query vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "target_field": { "type": "string", "description": "Name of the appended matches column (JSON array text). Defaults to 'matches'." }, "id_column": { "type": "string", "description": "Table column returned as the match id (as text). Defaults to 'id'." }, "vector_column": { "type": "string", "description": "Table column holding the stored vectors. Defaults to 'embedding'." }, "payload_column": { "type": "string", "description": "jsonb column included in each match as a payload object. Defaults to 'payload'; set to an empty string to disable." }, "metric": { "type": "string", "enum": [ "cosine", "l2", "inner_product" ], "description": "Distance operator: <=> (cosine, default), <-> (l2), <#> (inner product, negative)." }, "top_k": { "type": "integer", "description": "Number of neighbors per row. Defaults to 5." }, "concurrency": { "type": "integer", "description": "Maximum in-flight queries. Defaults to 4." }, "max_connections": { "type": "integer", "description": "Connection pool size. Defaults to 4." }, "timeout_ms": { "type": "integer", "description": "Pool acquire timeout in milliseconds. Defaults to 30000." } }, "required": [ "url", "table" ] }, "config_example": { "url": "postgres://postgres:postgres@localhost:5432/vectors", "table": "documents", "metric": "cosine", "top_k": 5 } }, { "kind": "processor", "name": "protobuf_to_arrow", "description": "Decodes Protobuf wire-format bytes into Arrow RecordBatches.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "message_type": { "type": "string", "description": "Fully-qualified Protobuf message type name." }, "proto_inputs": { "type": "array", "items": { "type": "string" }, "description": "Paths to .proto files." }, "proto_includes": { "type": "array", "items": { "type": "string" }, "description": "Include paths for proto resolution." }, "value_field": { "type": "string", "description": "Name of the binary column holding the Protobuf wire-format bytes (defaults to '__value')." } }, "required": [ "message_type", "proto_inputs" ] } }, { "kind": "processor", "name": "python", "description": "Runs a user-defined Python function (with PyArrow) against each batch.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "script": { "type": "string", "description": "Python source defining the transform function (optional when importing an existing module)." }, "module": { "type": "string", "default": "__main__", "description": "Python module providing the function." }, "function": { "type": "string", "description": "Name of the function to invoke for each batch." }, "python_path": { "type": "array", "items": { "type": "string" }, "default": [], "description": "Extra sys.path entries, earlier entries take precedence." }, "timeout_ms": { "type": "integer", "minimum": 1, "default": 60000, "description": "Upper bound for one UDF call in milliseconds; longer calls fail the batch." } }, "required": [ "function" ] }, "config_example": { "script": "def transform(batch):\n return batch", "function": "transform" } }, { "kind": "processor", "name": "sql", "description": "Runs a DataFusion SQL query against each batch. Supports window functions and joins against temporary tables.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "query": { "type": "string", "description": "SQL query to run on every batch." }, "table_name": { "type": "string", "description": "Name used for the batch table in the query (default 'flow')." }, "temporary_list": { "type": "array", "description": "Temporary tables to register before running the query.", "items": { "type": "object", "properties": { "name": { "type": "string" }, "table_name": { "type": "string" }, "key": { "type": "object", "properties": { "value": { "type": "string" } }, "required": [ "value" ] } }, "required": [ "name", "table_name", "key" ] } } }, "required": [ "query" ] }, "config_example": { "query": "SELECT *, __meta_source AS source FROM flow" } }, { "kind": "processor", "name": "vector_search", "description": "Searches a Qdrant collection for the top-k nearest neighbors of each row's vector and appends the matches as a JSON array text column.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "url": { "type": "string", "description": "Qdrant base URL, e.g. http://localhost:6333." }, "collection": { "type": "string", "description": "Collection to search." }, "vector_field": { "type": "string", "description": "Name of the query vector column (FixedSizeList/List of Float32). Defaults to 'embedding'." }, "target_field": { "type": "string", "description": "Name of the appended matches column (JSON array text). Defaults to 'matches'." }, "top_k": { "type": "integer", "description": "Number of neighbors per row. Defaults to 5." }, "score_threshold": { "type": "number", "description": "Minimum similarity score; omitted from the request when unset." }, "concurrency": { "type": "integer", "description": "Maximum in-flight requests. Defaults to 4." }, "api_key": { "type": "string", "description": "API key sent as 'Authorization: Bearer'; supports secret references." }, "timeout_ms": { "type": "integer", "description": "HTTP request timeout in milliseconds. Defaults to 30000." }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Extra HTTP headers." } }, "required": [ "url", "collection" ] }, "config_example": { "url": "http://localhost:6333", "collection": "documents", "vector_field": "embedding", "top_k": 5 } }, { "kind": "processor", "name": "vrl", "description": "Runs a Vector Remap Language (VRL) program against each batch for safe transformation and enrichment.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "statement": { "type": "string", "description": "VRL program source." }, "timezone": { "type": "string", "description": "Optional timezone for VRL timestamp operations (e.g. 'Asia/Shanghai', 'UTC', 'local'). Defaults to the platform local timezone; invalid values fall back to the default with a warning." } }, "required": [ "statement" ] }, "config_example": { "statement": ".message = parse_json!(.message)\n.timestamp = now()" } }, { "kind": "buffer", "name": "memory", "description": "In-memory buffer that releases a batch when it reaches capacity or after a timeout.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "capacity": { "type": "integer", "minimum": 1, "description": "Maximum number of messages to accumulate before releasing." }, "timeout": { "type": "string", "description": "Maximum time to wait before releasing a partial batch (humantime)." } }, "required": [ "capacity", "timeout" ] }, "config_example": { "capacity": 1000, "timeout": "5s" } }, { "kind": "buffer", "name": "session_window", "description": "Groups messages into sessions based on a maximum gap between messages. Supports SQL joins across sources.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "gap": { "type": "string", "description": "Maximum idle time before a session is closed (humantime)." }, "join": { "type": "object", "description": "Optional SQL join across input sources.", "properties": { "query": { "type": "string" }, "value_field": { "type": "string" }, "codec": { "type": "object" }, "thread_num": { "type": "integer", "minimum": 1 } } } }, "required": [ "gap" ] }, "config_example": { "gap": "30s" } }, { "kind": "buffer", "name": "sliding_window", "description": "Overlapping time windows that slide forward by a fixed interval.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "window_size": { "type": "integer", "minimum": 1, "description": "Number of messages included in each window." }, "interval": { "type": "string", "description": "Time interval between window emissions (humantime)." }, "slide_size": { "type": "integer", "minimum": 1, "description": "Number of messages to advance the window by on each emission." } }, "required": [ "window_size", "interval", "slide_size" ] }, "config_example": { "window_size": 100, "interval": "5s", "slide_size": 20 } }, { "kind": "buffer", "name": "tumbling_window", "description": "Fixed-size, non-overlapping time windows. Supports SQL joins across sources.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "interval": { "type": "string", "description": "Window duration (humantime)." }, "join": { "type": "object", "description": "Optional SQL join across input sources.", "properties": { "query": { "type": "string" }, "value_field": { "type": "string" }, "codec": { "type": "object" }, "thread_num": { "type": "integer", "minimum": 1 } } } }, "required": [ "interval" ] }, "config_example": { "interval": "1m" } }, { "kind": "codec", "name": "debezium_json", "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.", "config_optional": true, "config_schema": { "type": "object", "additionalProperties": false, "properties": {} } }, { "kind": "codec", "name": "json", "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.", "config_optional": true, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "on_error": { "type": "string", "enum": [ "fail", "skip" ], "default": "fail", "description": "Decode-error policy: `fail` fails the batch (default), `skip` isolates bad messages with a warning and decodes the rest." } } } }, { "kind": "codec", "name": "protobuf", "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "message_type": { "type": "string", "description": "Fully-qualified Protobuf message type name." }, "proto_inputs": { "type": "array", "items": { "type": "string" }, "description": "Paths to .proto files." }, "proto_includes": { "type": "array", "items": { "type": "string" }, "description": "Include paths for proto resolution." }, "on_error": { "type": "string", "enum": [ "fail", "skip" ], "default": "fail", "description": "Decode-error policy: `fail` fails the batch (default), `skip` isolates bad messages with a warning and decodes the rest." } }, "required": [ "message_type", "proto_inputs" ] } }, { "kind": "codec", "name": "schema_registry", "description": "Decodes Confluent wire-format Protobuf and Avro messages by resolving the schema id from a Schema Registry, with an optional subject compatibility gate.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "registry_url": { "type": "string", "description": "Confluent Schema Registry base URL." }, "message_type": { "type": "string", "description": "Fully-qualified Protobuf message type. Required for Protobuf schemas; omit for Avro." }, "subject": { "type": "string", "description": "Registry subject for the compatibility gate." }, "min_compatibility": { "type": "string", "enum": [ "none", "backward", "forward", "full" ], "description": "Minimum subject compatibility level enforced on the first decoded message. Requires `subject`." }, "auth": { "type": "object", "description": "Optional registry authentication.", "properties": { "type": { "type": "string", "enum": [ "basic", "bearer" ] }, "username": { "type": "string" }, "password": { "type": "string" }, "token": { "type": "string" } } } }, "required": [ "registry_url" ] } }, { "kind": "temporary", "name": "redis", "description": "Redis-backed temporary lookup store (single node or cluster) read through a codec.", "config_optional": false, "config_schema": { "type": "object", "additionalProperties": false, "properties": { "mode": { "description": "Redis connection mode.", "oneOf": [ { "type": "object", "title": "single", "properties": { "type": { "const": "single" }, "url": { "type": "string", "description": "Redis URL, e.g. redis://127.0.0.1:6379." } }, "required": [ "type", "url" ] }, { "type": "object", "title": "cluster", "properties": { "type": { "const": "cluster" }, "urls": { "type": "array", "items": { "type": "string" }, "description": "Redis cluster node URLs." } }, "required": [ "type", "urls" ] } ] }, "redis_type": { "description": "Redis structure holding the values.", "oneOf": [ { "type": "object", "title": "list", "properties": { "type": { "const": "list" } }, "required": [ "type" ] }, { "type": "object", "title": "string", "properties": { "type": { "const": "string" } }, "required": [ "type" ] } ] }, "codec": { "type": "object", "description": "Codec used to decode stored values." } }, "required": [ "mode", "redis_type", "codec" ] } } ] }