{
  "version": 2,
  "components": [
    {
      "kind": "input",
      "name": "file",
      "description": "Reads records from local or remote object storage (S3, GCS, Azure, HDFS) in CSV/JSON/Parquet/Avro/Arrow formats.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "input_type": {
            "type": "object",
            "description": "Format-specific input settings (type: csv/json/parquet/avro/arrow)."
          },
          "ballista": {
            "type": "object",
            "description": "Optional Ballista distributed compute configuration."
          },
          "query": {
            "type": "object",
            "description": "Optional SQL query and table name to filter the read."
          }
        },
        "required": [
          "input_type"
        ]
      },
      "config_example": {
        "input_type": {
          "type": "parquet",
          "path": "s3://bucket/data.parquet"
        }
      }
    },
    {
      "kind": "input",
      "name": "generate",
      "description": "Generates synthetic text messages on a fixed interval (useful for testing and load simulation).",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "context": {
            "type": "string",
            "description": "Payload string emitted on every read."
          },
          "interval": {
            "type": "string",
            "description": "Delay between batches (e.g. '100ms', '1s'). Accepts any humantime duration."
          },
          "count": {
            "type": "integer",
            "minimum": 1,
            "description": "Optional total number of messages to emit before signalling EOF."
          },
          "batch_size": {
            "type": "integer",
            "minimum": 1,
            "default": 1,
            "description": "Number of messages returned per read call."
          }
        },
        "required": [
          "context",
          "interval"
        ]
      },
      "config_example": {
        "context": "hello",
        "interval": "1s",
        "count": 100,
        "batch_size": 10
      }
    },
    {
      "kind": "input",
      "name": "http",
      "description": "Receives data via HTTP. Can run as a server (POST/PUT on `path`) or poll a remote endpoint.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "address": {
            "type": "string",
            "description": "Bind address (server mode) or base URL (client mode)."
          },
          "path": {
            "type": "string",
            "description": "Path to accept messages on (server mode) or full request path (client mode)."
          },
          "method": {
            "type": "string",
            "enum": [
              "GET",
              "POST",
              "PUT",
              "DELETE",
              "PATCH"
            ],
            "description": "HTTP method (client mode)."
          },
          "interval": {
            "type": "string",
            "description": "Polling interval in humantime format (client mode, e.g. '5s')."
          },
          "cors_enabled": {
            "type": "boolean",
            "default": false,
            "description": "Enable CORS for the server."
          },
          "auth": {
            "type": "object",
            "description": "Authentication configuration."
          }
        },
        "required": [
          "address",
          "path"
        ]
      },
      "config_example": {
        "address": "0.0.0.0:8080",
        "path": "/ingest",
        "cors_enabled": true
      }
    },
    {
      "kind": "input",
      "name": "kafka",
      "description": "Consumes messages from Apache Kafka topics with a consumer group.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "brokers": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "List of Kafka broker addresses."
          },
          "topics": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Topics to subscribe to."
          },
          "consumer_group": {
            "type": "string",
            "description": "Consumer group ID for offset coordination."
          },
          "client_id": {
            "type": "string",
            "description": "Optional client identifier."
          },
          "start_from_latest": {
            "type": "boolean",
            "default": false,
            "description": "When true, ignore committed offsets and start from the latest message."
          },
          "fetch_min_bytes": {
            "type": "integer",
            "minimum": 0,
            "description": "Minimum bytes before the broker responds to a fetch request."
          },
          "fetch_max_bytes": {
            "type": "integer",
            "minimum": 0,
            "description": "Maximum bytes for a fetch request."
          },
          "fetch_max_partition_bytes": {
            "type": "integer",
            "minimum": 0,
            "description": "Maximum bytes per partition in a fetch request."
          },
          "fetch_wait_max_ms": {
            "type": "integer",
            "minimum": 0,
            "description": "Maximum time to wait for fetch data in milliseconds."
          }
        },
        "required": [
          "brokers",
          "topics",
          "consumer_group"
        ]
      },
      "config_example": {
        "brokers": [
          "localhost:9092"
        ],
        "topics": [
          "events"
        ],
        "consumer_group": "arkflow"
      }
    },
    {
      "kind": "input",
      "name": "memory",
      "description": "In-memory input queue seeded with an initial list of messages. Primarily for tests and demos.",
      "config_optional": true,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "messages": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Initial messages to enqueue."
          }
        }
      },
      "config_example": {
        "messages": [
          "hello",
          "world"
        ]
      }
    },
    {
      "kind": "input",
      "name": "modbus",
      "description": "Polls Modbus TCP devices on a fixed interval, reading coils, discrete inputs, or registers.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "addr": {
            "type": "string",
            "description": "Modbus server address (host:port)."
          },
          "slave_id": {
            "type": "integer",
            "description": "Modbus unit / slave ID."
          },
          "points": {
            "type": "array",
            "description": "Points to read on every poll.",
            "items": {
              "type": "object",
              "properties": {
                "type": {
                  "type": "string",
                  "enum": [
                    "coils",
                    "discrete_inputs",
                    "holding_registers",
                    "input_registers"
                  ]
                },
                "name": {
                  "type": "string"
                },
                "address": {
                  "type": "integer"
                },
                "quantity": {
                  "type": "integer",
                  "minimum": 1
                }
              },
              "required": [
                "type",
                "name",
                "address",
                "quantity"
              ]
            }
          },
          "interval": {
            "type": "string",
            "description": "Poll interval (humantime)."
          }
        },
        "required": [
          "addr",
          "slave_id",
          "points",
          "interval"
        ]
      },
      "config_example": {
        "addr": "127.0.0.1:502",
        "slave_id": 1,
        "interval": "1s",
        "points": [
          {
            "type": "holding_registers",
            "name": "voltage",
            "address": 0,
            "quantity": 1
          }
        ]
      }
    },
    {
      "kind": "input",
      "name": "mqtt",
      "description": "Subscribes to an MQTT broker and forwards messages from the configured topics.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "host": {
            "type": "string",
            "description": "MQTT broker hostname."
          },
          "port": {
            "type": "integer",
            "minimum": 1,
            "maximum": 65535,
            "description": "MQTT broker port."
          },
          "client_id": {
            "type": "string",
            "description": "Unique client identifier."
          },
          "username": {
            "type": "string",
            "description": "Optional username."
          },
          "password": {
            "type": "string",
            "description": "Optional password."
          },
          "topics": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Topics to subscribe to (MQTT wildcards supported)."
          },
          "qos": {
            "type": "integer",
            "enum": [
              0,
              1,
              2
            ],
            "default": 0,
            "description": "Quality of Service level."
          },
          "clean_session": {
            "type": "boolean",
            "default": true,
            "description": "Whether to use a clean session."
          },
          "keep_alive": {
            "type": "integer",
            "minimum": 1,
            "description": "Keep-alive interval in seconds."
          }
        },
        "required": [
          "host",
          "port",
          "client_id",
          "topics"
        ]
      },
      "config_example": {
        "host": "localhost",
        "port": 1883,
        "client_id": "arkflow",
        "topics": [
          "sensors/#"
        ]
      }
    },
    {
      "kind": "input",
      "name": "multiple_inputs",
      "description": "Combines multiple input sources into a single stream. Each source is tagged with __meta_source.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "inputs": {
            "type": "array",
            "description": "List of input components to combine.",
            "items": {
              "type": "object",
              "properties": {
                "type": {
                  "type": "string",
                  "description": "Input type (e.g. 'kafka', 'http')."
                },
                "name": {
                  "type": "string",
                  "description": "Optional logical name for this source."
                }
              },
              "required": [
                "type"
              ]
            }
          }
        },
        "required": [
          "inputs"
        ]
      },
      "config_example": {
        "inputs": [
          {
            "type": "kafka",
            "name": "events",
            "topics": [
              "events"
            ]
          },
          {
            "type": "kafka",
            "name": "logs",
            "topics": [
              "logs"
            ]
          }
        ]
      }
    },
    {
      "kind": "input",
      "name": "nats",
      "description": "Consumes messages from NATS, supporting both regular subjects and JetStream consumers.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "url": {
            "type": "string",
            "description": "NATS server URL (e.g. nats://localhost:4222)."
          },
          "mode": {
            "type": "object",
            "description": "Select between plain NATS and JetStream subscriptions.",
            "oneOf": [
              {
                "properties": {
                  "type": {
                    "const": "regular"
                  },
                  "subject": {
                    "type": "string"
                  },
                  "queue_group": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "subject"
                ]
              },
              {
                "properties": {
                  "type": {
                    "const": "jet_stream"
                  },
                  "stream": {
                    "type": "string"
                  },
                  "consumer_name": {
                    "type": "string"
                  },
                  "durable_name": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "stream",
                  "consumer_name"
                ]
              }
            ]
          },
          "auth": {
            "type": "object",
            "description": "NATS authentication configuration."
          }
        },
        "required": [
          "url",
          "mode"
        ]
      },
      "config_example": {
        "url": "nats://localhost:4222",
        "mode": {
          "type": "regular",
          "subject": "events"
        }
      }
    },
    {
      "kind": "input",
      "name": "pulsar",
      "description": "Subscribes to an Apache Pulsar topic with configurable subscription type and authentication.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "service_url": {
            "type": "string",
            "description": "Pulsar service URL (e.g. pulsar://localhost:6650)."
          },
          "topic": {
            "type": "string",
            "description": "Topic to subscribe to."
          },
          "subscription_name": {
            "type": "string",
            "description": "Subscription name."
          },
          "subscription_type": {
            "type": "string",
            "enum": [
              "Exclusive",
              "Shared",
              "Failover",
              "KeyShared"
            ],
            "description": "Subscription type."
          },
          "auth": {
            "type": "object",
            "description": "Pulsar authentication configuration."
          },
          "retry_config": {
            "type": "object",
            "description": "Retry behaviour for failed messages."
          }
        },
        "required": [
          "service_url",
          "topic",
          "subscription_name"
        ]
      },
      "config_example": {
        "service_url": "pulsar://localhost:6650",
        "topic": "persistent://public/default/events",
        "subscription_name": "arkflow"
      }
    },
    {
      "kind": "input",
      "name": "redis",
      "description": "Reads from Redis: list blocking pops, pub/sub subscriptions, or stream consumer groups.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "mode": {
            "type": "object",
            "description": "Connection mode.",
            "oneOf": [
              {
                "properties": {
                  "type": {
                    "const": "cluster"
                  },
                  "urls": {
                    "type": "array",
                    "items": {
                      "type": "string"
                    }
                  }
                },
                "required": [
                  "type",
                  "urls"
                ]
              },
              {
                "properties": {
                  "type": {
                    "const": "single"
                  },
                  "url": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "url"
                ]
              }
            ]
          },
          "redis_type": {
            "type": "object",
            "description": "Data structure to consume from.",
            "oneOf": [
              {
                "properties": {
                  "type": {
                    "const": "subscribe"
                  },
                  "subscribe": {
                    "type": "object",
                    "oneOf": [
                      {
                        "properties": {
                          "type": {
                            "const": "channels"
                          },
                          "channels": {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          }
                        },
                        "required": [
                          "type",
                          "channels"
                        ]
                      },
                      {
                        "properties": {
                          "type": {
                            "const": "patterns"
                          },
                          "patterns": {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          }
                        },
                        "required": [
                          "type",
                          "patterns"
                        ]
                      }
                    ]
                  }
                },
                "required": [
                  "type",
                  "subscribe"
                ]
              },
              {
                "properties": {
                  "type": {
                    "const": "list"
                  },
                  "list": {
                    "type": "array",
                    "items": {
                      "type": "string"
                    }
                  }
                },
                "required": [
                  "type",
                  "list"
                ]
              }
            ]
          }
        },
        "required": [
          "mode",
          "redis_type"
        ]
      },
      "config_example": {
        "mode": {
          "type": "single",
          "url": "redis://localhost:6379"
        },
        "redis_type": {
          "type": "list",
          "list": [
            "events"
          ]
        }
      }
    },
    {
      "kind": "input",
      "name": "sql",
      "description": "Polls a SQL database (MySQL / PostgreSQL / SQLite / DuckDB) with a SELECT statement and emits rows as batches.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "select_sql": {
            "type": "string",
            "description": "SELECT statement to execute on every poll."
          },
          "poll_interval": {
            "type": "string",
            "description": "Optional poll interval (humantime)."
          },
          "ballista": {
            "type": "object",
            "description": "Optional Ballista distributed compute configuration."
          },
          "input_type": {
            "type": "object",
            "description": "Database connection settings."
          }
        },
        "required": [
          "select_sql",
          "input_type"
        ]
      },
      "config_example": {
        "select_sql": "SELECT id, name FROM users",
        "poll_interval": "10s",
        "input_type": {
          "type": "sqlite",
          "uri": "file:./data.db"
        }
      }
    },
    {
      "kind": "input",
      "name": "websocket",
      "description": "Connects to a WebSocket server and forwards each incoming message as a batch.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "url": {
            "type": "string",
            "description": "WebSocket server URL (ws:// or wss://)."
          },
          "headers": {
            "type": "object",
            "additionalProperties": {
              "type": "string"
            },
            "description": "Headers included in the WebSocket handshake."
          },
          "timeout": {
            "type": "integer",
            "minimum": 1,
            "description": "Connection timeout in seconds."
          }
        },
        "required": [
          "url"
        ]
      },
      "config_example": {
        "url": "ws://localhost:8080/stream"
      }
    },
    {
      "kind": "output",
      "name": "drop",
      "description": "Discards all messages. Useful for performance benchmarks and dead-end pipelines.",
      "config_optional": true,
      "config_schema": {
        "type": "object"
      }
    },
    {
      "kind": "output",
      "name": "http",
      "description": "Posts each batch to an HTTP endpoint. Supports custom headers, retry, and auth.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "url": {
            "type": "string",
            "description": "Destination URL."
          },
          "method": {
            "type": "string",
            "enum": [
              "GET",
              "POST",
              "PUT",
              "DELETE",
              "PATCH"
            ],
            "default": "POST",
            "description": "HTTP method."
          },
          "timeout_ms": {
            "type": "integer",
            "minimum": 1,
            "description": "Request timeout in milliseconds."
          },
          "retry_count": {
            "type": "integer",
            "minimum": 0,
            "description": "Number of retry attempts on failure."
          },
          "headers": {
            "type": "object",
            "additionalProperties": {
              "type": "string"
            },
            "description": "Custom HTTP headers."
          },
          "body_field": {
            "type": "string",
            "description": "Record field that holds the request body."
          },
          "auth": {
            "type": "object",
            "description": "Authentication configuration."
          }
        },
        "required": [
          "url"
        ]
      },
      "config_example": {
        "url": "https://example.com/ingest",
        "method": "POST",
        "timeout_ms": 5000,
        "retry_count": 3
      }
    },
    {
      "kind": "output",
      "name": "influxdb",
      "description": "Writes time-series data to InfluxDB v2.x using the Line Protocol.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "url": {
            "type": "string",
            "description": "InfluxDB server URL (e.g. http://localhost:8086)."
          },
          "org": {
            "type": "string",
            "description": "Organization name."
          },
          "bucket": {
            "type": "string",
            "description": "Destination bucket."
          },
          "token": {
            "type": "string",
            "description": "Authentication token."
          },
          "measurement": {
            "type": "string",
            "description": "Measurement name."
          },
          "tags": {
            "type": "array",
            "items": {
              "type": "object",
              "properties": {
                "field": {
                  "type": "string"
                },
                "tag_name": {
                  "type": "string"
                }
              },
              "required": [
                "field",
                "tag_name"
              ]
            },
            "description": "Tag mappings (label fields)."
          },
          "fields": {
            "type": "array",
            "items": {
              "type": "object",
              "properties": {
                "field": {
                  "type": "string"
                },
                "field_name": {
                  "type": "string"
                },
                "field_type": {
                  "enum": [
                    "float",
                    "integer",
                    "boolean",
                    "string"
                  ]
                }
              },
              "required": [
                "field",
                "field_name"
              ]
            },
            "description": "Field mappings (value fields)."
          },
          "timestamp_field": {
            "type": "string",
            "description": "Source field for the point timestamp."
          },
          "batch_size": {
            "type": "integer",
            "minimum": 1,
            "description": "Batch size for write requests."
          },
          "flush_interval": {
            "type": "integer",
            "minimum": 1,
            "description": "Maximum seconds to wait before flushing a partial batch."
          },
          "retry_count": {
            "type": "integer",
            "minimum": 1,
            "description": "Number of HTTP attempts per flush."
          },
          "timeout_ms": {
            "type": "integer",
            "minimum": 1,
            "description": "HTTP request timeout in milliseconds."
          }
        },
        "required": [
          "url",
          "org",
          "bucket",
          "token",
          "measurement",
          "fields"
        ]
      },
      "config_example": {
        "url": "http://localhost:8086",
        "org": "arkflow",
        "bucket": "metrics",
        "token": "${INFLUX_TOKEN}",
        "measurement": "sensor",
        "tags": [
          {
            "field": "sensor",
            "tag_name": "sensor"
          }
        ],
        "fields": [
          {
            "field": "value",
            "field_name": "value",
            "field_type": "float"
          }
        ],
        "timestamp_field": "timestamp"
      }
    },
    {
      "kind": "output",
      "name": "kafka",
      "description": "Produces messages to Apache Kafka. Supports key-based partitioning and compression.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "brokers": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "List of Kafka broker addresses."
          },
          "topic": {
            "type": "string",
            "description": "Destination topic (supports {field} placeholders)."
          },
          "key": {
            "type": "string",
            "description": "Field used as the message key for partitioning."
          },
          "client_id": {
            "type": "string",
            "description": "Optional client identifier."
          },
          "compression": {
            "type": "string",
            "enum": [
              "none",
              "gzip",
              "snappy",
              "lz4",
              "zstd"
            ],
            "description": "Compression algorithm."
          },
          "acks": {
            "type": "string",
            "enum": [
              "0",
              "1",
              "all"
            ],
            "description": "Acknowledgment level."
          },
          "value_field": {
            "type": "string",
            "description": "Record field used as the message payload."
          },
          "exactly_once": {
            "type": "boolean",
            "default": false,
            "description": "Enable exactly-once transactional production (L2)."
          },
          "transactional_id": {
            "type": "string",
            "description": "Transactional id (required when exactly_once is true); must be stable across restarts for zombie fencing."
          }
        },
        "required": [
          "brokers",
          "topic"
        ]
      },
      "config_example": {
        "brokers": [
          "localhost:9092"
        ],
        "topic": "events"
      }
    },
    {
      "kind": "output",
      "name": "mongodb",
      "description": "Writes Arrow rows to MongoDB as BSON documents.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "uri": {
            "type": "string",
            "description": "MongoDB connection URI."
          },
          "database": {
            "type": "string",
            "description": "Target database name."
          },
          "collection": {
            "type": "string",
            "description": "Target collection name."
          }
        },
        "required": [
          "uri",
          "database",
          "collection"
        ]
      },
      "config_example": {
        "uri": "mongodb://localhost:27017",
        "database": "arkflow",
        "collection": "events"
      }
    },
    {
      "kind": "output",
      "name": "mqtt",
      "description": "Publishes messages to an MQTT broker topic.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "host": {
            "type": "string",
            "description": "MQTT broker hostname."
          },
          "port": {
            "type": "integer",
            "minimum": 1,
            "maximum": 65535,
            "description": "MQTT broker port."
          },
          "client_id": {
            "type": "string",
            "description": "Client identifier."
          },
          "username": {
            "type": "string",
            "description": "Optional username."
          },
          "password": {
            "type": "string",
            "description": "Optional password."
          },
          "topic": {
            "type": "string",
            "description": "Destination topic (supports {field} placeholders)."
          },
          "qos": {
            "type": "integer",
            "enum": [
              0,
              1,
              2
            ],
            "default": 0,
            "description": "Quality of Service."
          },
          "clean_session": {
            "type": "boolean",
            "default": true
          },
          "keep_alive": {
            "type": "integer",
            "minimum": 1,
            "description": "Keep-alive interval in seconds."
          },
          "retain": {
            "type": "boolean",
            "default": false,
            "description": "Whether to retain the message on the broker."
          }
        },
        "required": [
          "host",
          "port",
          "client_id",
          "topic"
        ]
      },
      "config_example": {
        "host": "localhost",
        "port": 1883,
        "client_id": "arkflow",
        "topic": "sensors/data"
      }
    },
    {
      "kind": "output",
      "name": "nats",
      "description": "Publishes to NATS, either to a regular subject or a JetStream stream.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "url": {
            "type": "string",
            "description": "NATS server URL."
          },
          "mode": {
            "type": "object",
            "description": "Select regular or JetStream publishing.",
            "oneOf": [
              {
                "properties": {
                  "type": {
                    "const": "regular"
                  },
                  "subject": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "subject"
                ]
              },
              {
                "properties": {
                  "type": {
                    "const": "jet_stream"
                  },
                  "stream": {
                    "type": "string"
                  },
                  "subject": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "stream",
                  "subject"
                ]
              }
            ]
          },
          "auth": {
            "type": "object",
            "description": "NATS authentication configuration."
          },
          "value_field": {
            "type": "string",
            "description": "Record field used as the payload."
          }
        },
        "required": [
          "url",
          "mode"
        ]
      },
      "config_example": {
        "url": "nats://localhost:4222",
        "mode": {
          "type": "regular",
          "subject": "events"
        }
      }
    },
    {
      "kind": "output",
      "name": "pulsar",
      "description": "Produces messages to an Apache Pulsar topic.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "service_url": {
            "type": "string",
            "description": "Pulsar service URL."
          },
          "topic": {
            "type": "string",
            "description": "Destination topic (supports {field} placeholders)."
          },
          "auth": {
            "type": "object",
            "description": "Pulsar authentication configuration."
          },
          "value_field": {
            "type": "string",
            "description": "Record field used as the payload."
          }
        },
        "required": [
          "service_url",
          "topic"
        ]
      },
      "config_example": {
        "service_url": "pulsar://localhost:6650",
        "topic": "persistent://public/default/events"
      }
    },
    {
      "kind": "output",
      "name": "redis",
      "description": "Writes messages to Redis: streams, lists, or pub/sub channels.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "mode": {
            "type": "object",
            "description": "Connection mode (single or cluster).",
            "oneOf": [
              {
                "properties": {
                  "type": {
                    "const": "cluster"
                  },
                  "urls": {
                    "type": "array",
                    "items": {
                      "type": "string"
                    }
                  }
                },
                "required": [
                  "type",
                  "urls"
                ]
              },
              {
                "properties": {
                  "type": {
                    "const": "single"
                  },
                  "url": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "url"
                ]
              }
            ]
          },
          "redis_type": {
            "type": "object",
            "description": "Destination data structure.",
            "oneOf": [
              {
                "properties": {
                  "type": {
                    "const": "stream"
                  },
                  "stream": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "stream"
                ]
              },
              {
                "properties": {
                  "type": {
                    "const": "list"
                  },
                  "list": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "list"
                ]
              },
              {
                "properties": {
                  "type": {
                    "const": "channel"
                  },
                  "channel": {
                    "type": "string"
                  }
                },
                "required": [
                  "type",
                  "channel"
                ]
              }
            ]
          },
          "value_field": {
            "type": "string",
            "description": "Record field used as the payload."
          }
        },
        "required": [
          "mode",
          "redis_type"
        ]
      },
      "config_example": {
        "mode": {
          "type": "single",
          "url": "redis://localhost:6379"
        },
        "redis_type": {
          "type": "stream",
          "stream": "events"
        }
      }
    },
    {
      "kind": "output",
      "name": "sql",
      "description": "Batch-inserts records into a SQL database. Supports upsert and transaction management.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "connection": {
            "type": "object",
            "description": "Database connection settings (type, uri, etc.)."
          },
          "table": {
            "type": "string",
            "description": "Destination table."
          },
          "batch_size": {
            "type": "integer",
            "minimum": 1,
            "description": "Number of rows per insert batch."
          },
          "upsert": {
            "type": "boolean",
            "default": false,
            "description": "Use upsert (ON CONFLICT) instead of plain insert."
          },
          "upsert_keys": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Columns used to detect conflicts for upsert."
          }
        },
        "required": [
          "connection",
          "table"
        ]
      },
      "config_example": {
        "connection": {
          "type": "postgres",
          "uri": "postgres://user:pass@localhost/db"
        },
        "table": "events",
        "batch_size": 500
      }
    },
    {
      "kind": "output",
      "name": "stdout",
      "description": "Writes each message to the console. Useful for debugging and demos.",
      "config_optional": true,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "pretty": {
            "type": "boolean",
            "default": false,
            "description": "Pretty-print JSON output."
          }
        }
      },
      "config_example": {
        "pretty": true
      }
    },
    {
      "kind": "processor",
      "name": "arrow_to_json",
      "description": "Converts an Arrow RecordBatch into JSON byte payloads (one per row).",
      "config_optional": true,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "pretty": {
            "type": "boolean",
            "default": false,
            "description": "Pretty-print JSON output."
          }
        }
      }
    },
    {
      "kind": "processor",
      "name": "arrow_to_protobuf",
      "description": "Serializes Arrow RecordBatches into Protobuf wire-format bytes.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "message_type": {
            "type": "string",
            "description": "Fully-qualified Protobuf message type name."
          },
          "proto_inputs": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Paths to .proto files."
          },
          "proto_includes": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Include paths for proto resolution."
          },
          "fields_to_include": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Optional allow-list of field names to include when serializing to Protobuf."
          }
        },
        "required": [
          "message_type",
          "proto_inputs"
        ]
      }
    },
    {
      "kind": "processor",
      "name": "batch",
      "description": "Batches messages by count, size, or time interval before forwarding.",
      "config_optional": true,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "count": {
            "type": "integer",
            "minimum": 1,
            "description": "Maximum number of messages per batch."
          },
          "size": {
            "type": "integer",
            "minimum": 1,
            "description": "Approximate maximum byte size per batch."
          },
          "interval": {
            "type": "string",
            "description": "Maximum time to wait before flushing a partial batch (humantime)."
          }
        }
      },
      "config_example": {
        "count": 100,
        "interval": "5s"
      }
    },
    {
      "kind": "processor",
      "name": "json_to_arrow",
      "description": "Parses JSON byte payloads into an Arrow RecordBatch with inferred schema.",
      "config_optional": true,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "batch_size": {
            "type": "integer",
            "minimum": 1,
            "description": "Rows per output batch."
          }
        }
      }
    },
    {
      "kind": "processor",
      "name": "protobuf_to_arrow",
      "description": "Decodes Protobuf wire-format bytes into Arrow RecordBatches.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "message_type": {
            "type": "string",
            "description": "Fully-qualified Protobuf message type name."
          },
          "proto_inputs": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Paths to .proto files."
          },
          "proto_includes": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Include paths for proto resolution."
          },
          "value_field": {
            "type": "string",
            "description": "Name of the binary column holding the Protobuf wire-format bytes (defaults to '__value')."
          }
        },
        "required": [
          "message_type",
          "proto_inputs"
        ]
      }
    },
    {
      "kind": "processor",
      "name": "python",
      "description": "Runs a user-defined Python function (with PyArrow) against each batch.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "script": {
            "type": "string",
            "description": "Python source defining the transform function."
          },
          "function": {
            "type": "string",
            "description": "Name of the function to invoke for each batch."
          },
          "extra_packages": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Optional list of pip packages to install before running."
          }
        },
        "required": [
          "script",
          "function"
        ]
      },
      "config_example": {
        "script": "def transform(batch):\n    return batch",
        "function": "transform"
      }
    },
    {
      "kind": "processor",
      "name": "sql",
      "description": "Runs a DataFusion SQL query against each batch. Supports window functions and joins against temporary tables.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "query": {
            "type": "string",
            "description": "SQL query to run on every batch."
          },
          "table_name": {
            "type": "string",
            "description": "Name used for the batch table in the query (default 'flow')."
          },
          "temporary_list": {
            "type": "array",
            "description": "Temporary tables to register before running the query.",
            "items": {
              "type": "object",
              "properties": {
                "name": {
                  "type": "string"
                },
                "table_name": {
                  "type": "string"
                },
                "key": {
                  "type": "object",
                  "properties": {
                    "value": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "value"
                  ]
                }
              },
              "required": [
                "name",
                "table_name",
                "key"
              ]
            }
          }
        },
        "required": [
          "query"
        ]
      },
      "config_example": {
        "query": "SELECT *, __meta_source AS source FROM flow"
      }
    },
    {
      "kind": "processor",
      "name": "vrl",
      "description": "Runs a Vector Remap Language (VRL) program against each batch for safe transformation and enrichment.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "statement": {
            "type": "string",
            "description": "VRL program source."
          },
          "timezone": {
            "type": "string",
            "description": "Optional timezone for VRL timestamp operations (e.g. 'Asia/Shanghai', 'UTC', 'local'). Defaults to the platform local timezone; invalid values fall back to the default with a warning."
          }
        },
        "required": [
          "statement"
        ]
      },
      "config_example": {
        "statement": ".message = parse_json!(.message)\n.timestamp = now()"
      }
    },
    {
      "kind": "buffer",
      "name": "memory",
      "description": "In-memory buffer that releases a batch when it reaches capacity or after a timeout.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "capacity": {
            "type": "integer",
            "minimum": 1,
            "description": "Maximum number of messages to accumulate before releasing."
          },
          "timeout": {
            "type": "string",
            "description": "Maximum time to wait before releasing a partial batch (humantime)."
          }
        },
        "required": [
          "capacity",
          "timeout"
        ]
      },
      "config_example": {
        "capacity": 1000,
        "timeout": "5s"
      }
    },
    {
      "kind": "buffer",
      "name": "session_window",
      "description": "Groups messages into sessions based on a maximum gap between messages. Supports SQL joins across sources.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "gap": {
            "type": "string",
            "description": "Maximum idle time before a session is closed (humantime)."
          },
          "join": {
            "type": "object",
            "description": "Optional SQL join across input sources.",
            "properties": {
              "query": {
                "type": "string"
              },
              "value_field": {
                "type": "string"
              },
              "codec": {
                "type": "object"
              },
              "thread_num": {
                "type": "integer",
                "minimum": 1
              }
            }
          }
        },
        "required": [
          "gap"
        ]
      },
      "config_example": {
        "gap": "30s"
      }
    },
    {
      "kind": "buffer",
      "name": "sliding_window",
      "description": "Overlapping time windows that slide forward by a fixed interval.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "window_size": {
            "type": "integer",
            "minimum": 1,
            "description": "Number of messages included in each window."
          },
          "interval": {
            "type": "string",
            "description": "Time interval between window emissions (humantime)."
          },
          "slide_size": {
            "type": "integer",
            "minimum": 1,
            "description": "Number of messages to advance the window by on each emission."
          }
        },
        "required": [
          "window_size",
          "interval",
          "slide_size"
        ]
      },
      "config_example": {
        "window_size": 100,
        "interval": "5s",
        "slide_size": 20
      }
    },
    {
      "kind": "buffer",
      "name": "tumbling_window",
      "description": "Fixed-size, non-overlapping time windows. Supports SQL joins across sources.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "interval": {
            "type": "string",
            "description": "Window duration (humantime)."
          },
          "join": {
            "type": "object",
            "description": "Optional SQL join across input sources.",
            "properties": {
              "query": {
                "type": "string"
              },
              "value_field": {
                "type": "string"
              },
              "codec": {
                "type": "object"
              },
              "thread_num": {
                "type": "integer",
                "minimum": 1
              }
            }
          }
        },
        "required": [
          "interval"
        ]
      },
      "config_example": {
        "interval": "1m"
      }
    },
    {
      "kind": "codec",
      "name": "debezium_json",
      "description": "Decodes Debezium CDC Envelope JSON (before/after/op/source/ts_ms) into a columnar Arrow batch; attach to a Kafka input consuming a Debezium topic. CDC offset is the Kafka input's ack-gated offset.",
      "config_optional": true,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {}
      }
    },
    {
      "kind": "codec",
      "name": "json",
      "description": "Encodes/decodes Arrow RecordBatches as JSON byte payloads.",
      "config_optional": true,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "pretty": {
            "type": "boolean",
            "default": false,
            "description": "Pretty-print JSON output."
          }
        }
      }
    },
    {
      "kind": "codec",
      "name": "protobuf",
      "description": "Encodes/decodes Arrow RecordBatches using a Protobuf descriptor.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "message_type": {
            "type": "string",
            "description": "Fully-qualified Protobuf message type name."
          },
          "proto_inputs": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Paths to .proto files."
          },
          "proto_includes": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Include paths for proto resolution."
          }
        },
        "required": [
          "message_type",
          "proto_inputs"
        ]
      }
    },
    {
      "kind": "codec",
      "name": "schema_registry",
      "description": "Decodes Confluent wire-format Protobuf messages by resolving the schema id from a Schema Registry.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "registry_url": {
            "type": "string",
            "description": "Confluent Schema Registry base URL."
          },
          "message_type": {
            "type": "string",
            "description": "Fully-qualified Protobuf message type."
          },
          "auth": {
            "type": "object",
            "description": "Optional registry authentication.",
            "properties": {
              "type": {
                "type": "string",
                "enum": [
                  "basic",
                  "bearer"
                ]
              },
              "username": {
                "type": "string"
              },
              "password": {
                "type": "string"
              },
              "token": {
                "type": "string"
              }
            }
          }
        },
        "required": [
          "registry_url",
          "message_type"
        ]
      }
    },
    {
      "kind": "temporary",
      "name": "redis",
      "description": "Redis-backed temporary lookup store (single node or cluster) read through a codec.",
      "config_optional": false,
      "config_schema": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "mode": {
            "description": "Redis connection mode.",
            "oneOf": [
              {
                "type": "object",
                "title": "single",
                "properties": {
                  "type": {
                    "const": "single"
                  },
                  "url": {
                    "type": "string",
                    "description": "Redis URL, e.g. redis://127.0.0.1:6379."
                  }
                },
                "required": [
                  "type",
                  "url"
                ]
              },
              {
                "type": "object",
                "title": "cluster",
                "properties": {
                  "type": {
                    "const": "cluster"
                  },
                  "urls": {
                    "type": "array",
                    "items": {
                      "type": "string"
                    },
                    "description": "Redis cluster node URLs."
                  }
                },
                "required": [
                  "type",
                  "urls"
                ]
              }
            ]
          },
          "redis_type": {
            "description": "Redis structure holding the values.",
            "oneOf": [
              {
                "type": "object",
                "title": "list",
                "properties": {
                  "type": {
                    "const": "list"
                  }
                },
                "required": [
                  "type"
                ]
              },
              {
                "type": "object",
                "title": "string",
                "properties": {
                  "type": {
                    "const": "string"
                  }
                },
                "required": [
                  "type"
                ]
              }
            ]
          },
          "codec": {
            "type": "object",
            "description": "Codec used to decode stored values."
          }
        },
        "required": [
          "mode",
          "redis_type",
          "codec"
        ]
      }
    }
  ]
}