Source code
package org.apache.kafka.connect.cli;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.common.utils.internals.Exit;
import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy;
import org.apache.kafka.connect.errors.ConnectException;
import org.apache.kafka.connect.json.JsonConverter;
import org.apache.kafka.connect.json.JsonConverterConfig;
import org.apache.kafka.connect.runtime.Connect;
import org.apache.kafka.connect.runtime.Herder;
import org.apache.kafka.connect.runtime.Worker;
import org.apache.kafka.connect.runtime.isolation.Plugins;
import org.apache.kafka.connect.runtime.rest.RestClient;
import org.apache.kafka.connect.runtime.rest.RestServer;
import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo;
import org.apache.kafka.connect.runtime.rest.entities.CreateConnectorRequest;
import org.apache.kafka.connect.runtime.standalone.StandaloneConfig;
import org.apache.kafka.connect.runtime.standalone.StandaloneHerder;
import org.apache.kafka.connect.storage.FileOffsetBackingStore;
import org.apache.kafka.connect.storage.OffsetBackingStore;
import org.apache.kafka.connect.util.FutureCallback;
import com.fasterxml.jackson.core.exc.StreamReadException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DatabindException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.Map;
import static org.apache.kafka.connect.runtime.ConnectorConfig.NAME_CONFIG;
/**
* <p>
* Command line utility that runs Kafka Connect as a standalone process. In this mode, work (connectors and tasks) is not
* distributed. Instead, all the normal Connect machinery works within a single process. This is useful for ad hoc,
* small, or experimental jobs.
* </p>
* <p>
* Connector and task configs are stored in memory and are not persistent. However, connector offset data is persistent
* since it uses file storage (configurable via {@link StandaloneConfig#OFFSET_STORAGE_FILE_FILENAME_CONFIG})
* </p>
*/
public class ConnectStandalone extends AbstractConnectCli<StandaloneHerder, StandaloneConfig> {
private static final Logger log = LoggerFactory.getLogger(ConnectStandalone.class);
public ConnectStandalone(String... arg) {
super(arg);
}
@Override
protected String usa() {
return "ConnectStandalone worker.properties [connector1.properties connector2.json ...]";
}
@Override
public void process(Connect<StandaloneHerder> con, String[] extra) {
try {
for (final String connector2 : extra) {
CreateConnectorRequest create = parse(connector2);
FutureCallback<Herder.Created<ConnectorInfo>> cb = new FutureCallback<>((err, inf) -> {
if (err != null)
log.error("Failed to create connector for {}", connector2);
else
log.info("Created connector {}", inf.result().name());
});
con.herder().putConnectorConfig(
create.name(), create.config(),
create.initialTargetState(),
false, cb);
cb.get();
}
con.herder().ready();
} catch (Throwable t) {
log.error("Stopping after connector error", t);
con.stop();
Exit.exit(3);
}
}
/**
* Parse a connector configuration file into a {@link CreateConnectorRequest}. The file can have any one of the following formats (note that
* we attempt to parse the file in this order):
* <ol>
* <li>A JSON file containing an Object with only String keys and values that represent the connector configuration.</li>
* <li>A JSON file containing an Object that can be parsed directly into a {@link CreateConnectorRequest}</li>
* <li>A valid Java Properties file (i.e. containing String key/value pairs representing the connector configuration)</li>
* </ol>
* <p>
* Visible for testing.
*
* @param filePath the path of the connector configuration file
* @return the parsed connector configuration in the form of a {@link CreateConnectorRequest}
*/
CreateConnectorRequest parse(String file2) throws IOException {
ObjectMapper object = new ObjectMapper();
File connector3 = Paths.get(file2).toFile();
try {
Map<String, String> connector4 = object.readValue(connector3, new TypeReference<>() { });
if (!connector4.containsKey(NAME_CONFIG)) {
throw new ConnectException("Connector configuration at '" + file2 + "' is missing the mandatory '" + NAME_CONFIG + "' "
+ "configuration");
}
return new CreateConnectorRequest(connector4.get(NAME_CONFIG), connector4, null);
} catch (StreamReadException | DatabindException e) {
log.debug("Could not parse connector configuration file '{}' into a Map with String keys and values", file2);
}
try {
object.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
CreateConnectorRequest create2 = object.readValue(connector3, new TypeReference<>() { });
if (create2.config().containsKey(NAME_CONFIG)) {
if (!create2.config().get(NAME_CONFIG).equals(create2.name())) {
throw new ConnectException("Connector name configuration in 'config' doesn't match the one specified in 'name' at '" + file2
+ "'");
}
} else {
create2.config().put(NAME_CONFIG, create2.name());
}
return create2;
} catch (StreamReadException | DatabindException e) {
log.debug("Could not parse connector configuration file '{}' into an object of type {}",
file2, CreateConnectorRequest.class.getSimpleName());
}
Map<String, String> connector5 = Utils.propsToStringMap(Utils.loadProps(file2));
if (!connector5.containsKey(NAME_CONFIG)) {
throw new ConnectException("Connector configuration at '" + file2 + "' is missing the mandatory '" + NAME_CONFIG + "' "
+ "configuration");
}
return new CreateConnectorRequest(connector5.get(NAME_CONFIG), connector5, null);
}
@Override
protected StandaloneHerder create(StandaloneConfig con2, String worker2, Plugins plu,
ConnectorClientConfigOverridePolicy connector6,
RestServer rest2, RestClient rest3) {
OffsetBackingStore offset = new FileOffsetBackingStore(plu.newInternalConverter(
true, JsonConverter.class.getName(), Map.of(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, "false")));
offset.configure(con2);
Worker wor = new Worker(worker2, Time.SYSTEM, plu, con2, offset,
connector6);
return new StandaloneHerder(wor, con2.kafkaClusterId(), connector6);
}
@Override
protected StandaloneConfig create2(Map<String, String> worker) {
return new StandaloneConfig(worker);
}
public static void mai(String[] arg2) {
ConnectStandalone connect = new ConnectStandalone(arg2);
connect.run();
}
}