Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ public class NiFiProperties extends ApplicationProperties {
public static final String LOAD_BALANCE_CONNECTIONS_PER_NODE = "nifi.cluster.load.balance.connections.per.node";
public static final String LOAD_BALANCE_MAX_THREAD_COUNT = "nifi.cluster.load.balance.max.thread.count";
public static final String LOAD_BALANCE_COMMS_TIMEOUT = "nifi.cluster.load.balance.comms.timeout";
public static final String LOAD_BALANCE_CONNECTION_TIMEOUT = "nifi.cluster.load.balance.connection.timeout";

// zookeeper properties
public static final String ZOOKEEPER_CONNECT_STRING = "nifi.zookeeper.connect.string";
Expand Down Expand Up @@ -434,6 +435,7 @@ public class NiFiProperties extends ApplicationProperties {
public static final int DEFAULT_LOAD_BALANCE_CONNECTIONS_PER_NODE = 4;
public static final int DEFAULT_LOAD_BALANCE_MAX_THREAD_COUNT = 8;
public static final String DEFAULT_LOAD_BALANCE_COMMS_TIMEOUT = "30 sec";
public static final String DEFAULT_LOAD_BALANCE_CONNECTION_TIMEOUT = "5 sec";

// state management defaults
public static final String DEFAULT_STATE_MANAGEMENT_CONFIG_FILE = "conf/state-management.xml";
Expand Down
1 change: 1 addition & 0 deletions nifi-docs/src/main/asciidoc/administration-guide.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -3772,6 +3772,7 @@ Subject Alternative Names included on the server certificate configured in the a

*NOTE:* Increasing this value will allow additional threads to be used for communicating with other nodes in the cluster and writing the data to the Content and FlowFile Repositories. However, if this property is set to a value greater than the number of nodes in the cluster multiplied by the number of connections per node (`nifi.cluster.load.balance.connections.per.node`), then no further benefit will be gained and resources will be wasted.
|`nifi.cluster.load.balance.comms.timeout`|When communicating with another node, if this amount of time elapses without making any progress when reading from or writing to a socket, then a TimeoutException will be thrown. This will then result in the data either being retried or sent to another node in the cluster, depending on the configured Load Balancing Strategy. The default value is `30 sec`.
|`nifi.cluster.load.balance.connection.timeout`|The maximum amount of time to wait when establishing a connection to another node in the cluster for load balancing. If the connection cannot be established within this time, the connection attempt fails. Keeping this value low limits how long data transfers to other nodes are delayed when a node is unreachable, for example after it has been restarted with a new network address. The default value is `5 sec`.
|====

=== ZooKeeper Properties
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -884,8 +884,10 @@ private FlowController(
numThreads, loadBalanceProtocol, eventReporter, timeoutMillis);

final int connectionsPerNode = nifiProperties.getIntegerProperty(NiFiProperties.LOAD_BALANCE_CONNECTIONS_PER_NODE, NiFiProperties.DEFAULT_LOAD_BALANCE_CONNECTIONS_PER_NODE);
final NioAsyncLoadBalanceClientFactory asyncClientFactory = new NioAsyncLoadBalanceClientFactory(sslContext, timeoutMillis, new ContentRepositoryFlowFileAccess(contentRepository),
eventReporter, new StandardLoadBalanceFlowFileCodec(), clusterCoordinator);
final String connectTimeoutPeriod = nifiProperties.getProperty(NiFiProperties.LOAD_BALANCE_CONNECTION_TIMEOUT, NiFiProperties.DEFAULT_LOAD_BALANCE_CONNECTION_TIMEOUT);
final int connectTimeoutMillis = (int) FormatUtils.getTimeDuration(connectTimeoutPeriod, TimeUnit.MILLISECONDS);
final NioAsyncLoadBalanceClientFactory asyncClientFactory = new NioAsyncLoadBalanceClientFactory(sslContext, timeoutMillis, connectTimeoutMillis,
new ContentRepositoryFlowFileAccess(contentRepository), eventReporter, new StandardLoadBalanceFlowFileCodec(), clusterCoordinator);
loadBalanceClientRegistry = new NioAsyncLoadBalanceClientRegistry(asyncClientFactory, connectionsPerNode);

final int loadBalanceClientThreadCount = nifiProperties.getIntegerProperty(NiFiProperties.LOAD_BALANCE_MAX_THREAD_COUNT, NiFiProperties.DEFAULT_LOAD_BALANCE_MAX_THREAD_COUNT);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ public class NioAsyncLoadBalanceClient implements AsyncLoadBalanceClient {

private final NodeIdentifier nodeIdentifier;
private final int timeoutMillis;
private final int connectTimeoutMillis;
private final FlowFileContentAccess flowFileContentAccess;
private final LoadBalanceFlowFileCodec flowFileCodec;
private final EventReporter eventReporter;
Expand All @@ -88,10 +89,12 @@ public class NioAsyncLoadBalanceClient implements AsyncLoadBalanceClient {
private final Lock loadBalanceSessionLock = new ReentrantLock();
private LoadBalanceSession loadBalanceSession = null;

public NioAsyncLoadBalanceClient(final NodeIdentifier nodeIdentifier, final SSLContext sslContext, final int timeoutMillis, final FlowFileContentAccess flowFileContentAccess,
final LoadBalanceFlowFileCodec flowFileCodec, final EventReporter eventReporter, final ClusterCoordinator clusterCoordinator) {
public NioAsyncLoadBalanceClient(final NodeIdentifier nodeIdentifier, final SSLContext sslContext, final int timeoutMillis, final int connectTimeoutMillis,
final FlowFileContentAccess flowFileContentAccess, final LoadBalanceFlowFileCodec flowFileCodec, final EventReporter eventReporter,
final ClusterCoordinator clusterCoordinator) {
this.nodeIdentifier = nodeIdentifier;
this.timeoutMillis = timeoutMillis;
this.connectTimeoutMillis = connectTimeoutMillis;
this.flowFileContentAccess = flowFileContentAccess;
this.flowFileCodec = flowFileCodec;
this.eventReporter = eventReporter;
Expand Down Expand Up @@ -486,7 +489,9 @@ private SocketChannel createChannel() throws IOException {
final Socket socket = socketChannel.socket();
socket.setSoTimeout(timeoutMillis);

socket.connect(new InetSocketAddress(nodeIdentifier.getLoadBalanceAddress(), nodeIdentifier.getLoadBalancePort()));
// Bound the connect: without a timeout, connecting to a peer that silently drops packets (e.g. the old address of a
// restarted node) blocks for the OS TCP connect timeout, stalling every other Client served by the same task thread
socket.connect(new InetSocketAddress(nodeIdentifier.getLoadBalanceAddress(), nodeIdentifier.getLoadBalancePort()), connectTimeoutMillis);
socket.setSoTimeout(timeoutMillis);

return socketChannel;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,18 @@
public class NioAsyncLoadBalanceClientFactory implements AsyncLoadBalanceClientFactory {
private final SSLContext sslContext;
private final int timeoutMillis;
private final int connectTimeoutMillis;
private final FlowFileContentAccess flowFileContentAccess;
private final EventReporter eventReporter;
private final LoadBalanceFlowFileCodec flowFileCodec;
private final ClusterCoordinator clusterCoordinator;

public NioAsyncLoadBalanceClientFactory(final SSLContext sslContext, final int timeoutMillis, final FlowFileContentAccess flowFileContentAccess, final EventReporter eventReporter,
public NioAsyncLoadBalanceClientFactory(final SSLContext sslContext, final int timeoutMillis, final int connectTimeoutMillis,
final FlowFileContentAccess flowFileContentAccess, final EventReporter eventReporter,
final LoadBalanceFlowFileCodec loadBalanceFlowFileCodec, final ClusterCoordinator clusterCoordinator) {
this.sslContext = sslContext;
this.timeoutMillis = timeoutMillis;
this.connectTimeoutMillis = connectTimeoutMillis;
this.flowFileContentAccess = flowFileContentAccess;
this.eventReporter = eventReporter;
this.flowFileCodec = loadBalanceFlowFileCodec;
Expand All @@ -46,6 +49,6 @@ public NioAsyncLoadBalanceClientFactory(final SSLContext sslContext, final int t

@Override
public NioAsyncLoadBalanceClient createClient(final NodeIdentifier nodeIdentifier) {
return new NioAsyncLoadBalanceClient(nodeIdentifier, sslContext, timeoutMillis, flowFileContentAccess, flowFileCodec, eventReporter, clusterCoordinator);
return new NioAsyncLoadBalanceClient(nodeIdentifier, sslContext, timeoutMillis, connectTimeoutMillis, flowFileContentAccess, flowFileCodec, eventReporter, clusterCoordinator);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ private ContentClaim createContentClaim(final byte[] bytes) {

private NioAsyncLoadBalanceClientFactory createClientFactory(final SSLContext sslContext) {
final FlowFileContentAccess flowFileContentAccess = flowFile -> clientContentRepo.read(flowFile.getContentClaim());
return new NioAsyncLoadBalanceClientFactory(sslContext, 30000, flowFileContentAccess, eventReporter, new StandardLoadBalanceFlowFileCodec(), clusterCoordinator);
return new NioAsyncLoadBalanceClientFactory(sslContext, 30000, 5000, flowFileContentAccess, eventReporter, new StandardLoadBalanceFlowFileCodec(), clusterCoordinator);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ void testUnregisterOfInFlightTransactionClosesChannel() throws Exception {

final FlowFileContentAccess contentAccess = ff -> new ByteArrayInputStream(new byte[0]);

final NioAsyncLoadBalanceClient client = new NioAsyncLoadBalanceClient(nodeId, null, 30000, contentAccess,
final NioAsyncLoadBalanceClient client = new NioAsyncLoadBalanceClient(nodeId, null, 30000, 5000, contentAccess,
new StandardLoadBalanceFlowFileCodec(), EventReporter.NO_OP, clusterCoordinator);

final String connectionId = "unit-test-connection";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public boolean isRebalanceOnFailure() {
void setUp() {
createdClients = new ArrayList<>();

final NioAsyncLoadBalanceClientFactory factory = new NioAsyncLoadBalanceClientFactory(null, 30000, null, null, null, null) {
final NioAsyncLoadBalanceClientFactory factory = new NioAsyncLoadBalanceClientFactory(null, 30000, 5000, null, null, null, null) {
@Override
public NioAsyncLoadBalanceClient createClient(final NodeIdentifier nodeIdentifier) {
final StubAsyncLoadBalanceClient client = new StubAsyncLoadBalanceClient(nodeIdentifier);
Expand Down Expand Up @@ -165,7 +165,7 @@ private static class StubAsyncLoadBalanceClient extends NioAsyncLoadBalanceClien
boolean stopped = false;

StubAsyncLoadBalanceClient(final NodeIdentifier nodeIdentifier) {
super(nodeIdentifier, null, 30000, null, null, null, null);
super(nodeIdentifier, null, 30000, 5000, null, null, null, null);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@
<nifi.cluster.load.balance.connections.per.node>1</nifi.cluster.load.balance.connections.per.node>
<nifi.cluster.load.balance.max.thread.count>8</nifi.cluster.load.balance.max.thread.count>
<nifi.cluster.load.balance.comms.timeout>30 sec</nifi.cluster.load.balance.comms.timeout>
<nifi.cluster.load.balance.connection.timeout>5 sec</nifi.cluster.load.balance.connection.timeout>

<!-- nifi.properties: zookeeper properties -->
<nifi.zookeeper.connect.string />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ nifi.cluster.load.balance.port=${nifi.cluster.load.balance.port}
nifi.cluster.load.balance.connections.per.node=${nifi.cluster.load.balance.connections.per.node}
nifi.cluster.load.balance.max.thread.count=${nifi.cluster.load.balance.max.thread.count}
nifi.cluster.load.balance.comms.timeout=${nifi.cluster.load.balance.comms.timeout}
nifi.cluster.load.balance.connection.timeout=${nifi.cluster.load.balance.connection.timeout}

# zookeeper properties, used for cluster management #
nifi.zookeeper.connect.string=${nifi.zookeeper.connect.string}
Expand Down