rdev/gossip-revisited
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed build

Browse Source
This commit is contained in:
Jaime Freire
2023-11-25 09:11:39 +01:00
parent 789a5f6abc
commit c0d8cc348c
71 changed files with 1426 additions and 1197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
gossip-base/src/main/java/org/apache/gossip/GossipSettings.java
View File

@ -21,10 +21,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.gossip.lock.LockManagerSettings;
/**
* In this object the settings used by the GossipService are held.
*
*/
/** In this object the settings used by the GossipService are held. */
public class GossipSettings {
/** Time between gossip'ing in ms. Default is 1 second. */
@ -49,7 +46,7 @@ public class GossipSettings {
private String transportManagerClass = "org.apache.gossip.transport.udp.UdpTransportManager";
private String protocolManagerClass = "org.apache.gossip.protocol.json.JacksonProtocolManager";
private Map<String,String> activeGossipProperties = new HashMap<>();
private Map<String, String> activeGossipProperties = new HashMap<>();
private String pathToRingState = "./";
@ -64,29 +61,30 @@ public class GossipSettings {
private boolean signMessages = false;
// Settings related to lock manager
private LockManagerSettings lockManagerSettings = LockManagerSettings
.getLockManagerDefaultSettings();
private LockManagerSettings lockManagerSettings =
LockManagerSettings.getLockManagerDefaultSettings();
private boolean bulkTransfer = false;
private int bulkTransferSize = StartupSettings.DEFAULT_BULK_TRANSFER_SIZE;
/**
* Construct GossipSettings with default settings.
*/
public GossipSettings() {
}
/** Construct GossipSettings with default settings. */
public GossipSettings() {}
/**
* Construct GossipSettings with given settings.
*
* @param gossipInterval
* The gossip interval in ms.
* @param cleanupInterval
* The cleanup interval in ms.
* @param gossipInterval The gossip interval in ms.
* @param cleanupInterval The cleanup interval in ms.
*/
public GossipSettings(int gossipInterval, int cleanupInterval, int windowSize, int minimumSamples,
double convictThreshold, String distribution, boolean bulkTransfer) {
public GossipSettings(
int gossipInterval,
int cleanupInterval,
int windowSize,
int minimumSamples,
double convictThreshold,
String distribution,
boolean bulkTransfer) {
this.gossipInterval = gossipInterval;
this.cleanupInterval = cleanupInterval;
this.windowSize = windowSize;
@ -99,8 +97,7 @@ public class GossipSettings {
/**
* Set the gossip interval. This is the time between a gossip message is send.
*
* @param gossipInterval
* The gossip interval in ms.
* @param gossipInterval The gossip interval in ms.
*/
public void setGossipTimeout(int gossipInterval) {
this.gossipInterval = gossipInterval;
@ -132,8 +129,7 @@ public class GossipSettings {
* Set the cleanup interval. This is the time between the last heartbeat received from a member
* and when it will be marked as dead.
*
* @param cleanupInterval
* The cleanup interval in ms.
* @param cleanupInterval The cleanup interval in ms.
*/
public void setCleanupInterval(int cleanupInterval) {
this.cleanupInterval = cleanupInterval;
@ -257,6 +253,7 @@ public class GossipSettings {
/**
* Set the lock settings use by the lock manager
*
* @param lockManagerSettings lock settings. This object cannot be null.
*/
public void setLockManagerSettings(LockManagerSettings lockManagerSettings) {

1
gossip-base/src/main/java/org/apache/gossip/LocalMember.java
View File

@ -19,7 +19,6 @@ package org.apache.gossip;
import java.net.URI;
import java.util.Map;
import org.apache.gossip.accrual.FailureDetector;
/**

97
gossip-base/src/main/java/org/apache/gossip/StartupSettings.java
View File

@ -32,34 +32,33 @@ import java.util.Map;
import java.util.Map.Entry;
import lombok.extern.slf4j.Slf4j;
/**
* This object represents the settings used when starting the gossip service.
*
*/
/** This object represents the settings used when starting the gossip service. */
@Slf4j
public class StartupSettings {
public static final int DEFAULT_BULK_TRANSFER_SIZE = 100;
/** Default setting values */
private static final boolean DEFAULT_BULK_TRANSFER = false;
/** The gossip settings used at startup. */
private final GossipSettings gossipSettings;
/** The list with gossip members to start with. */
private final List<Member> gossipMembers;
/** The id to use fo the service */
private String id;
private URI uri;
private String cluster;
/**
* Constructor.
*
* @param id
* The id to be used for this service
* @param uri
* A URI object containing IP/hostname and port
* @param logLevel
* unused
* @param id The id to be used for this service
* @param uri A URI object containing IP/hostname and port
* @param logLevel unused
*/
public StartupSettings(String id, URI uri, int logLevel, String cluster) {
this(id, uri, new GossipSettings(), cluster);
@ -68,10 +67,8 @@ public class StartupSettings {
/**
* Constructor.
*
* @param id
* The id to be used for this service
* @param uri
* A URI object containing IP/hostname and port
* @param id The id to be used for this service
* @param uri A URI object containing IP/hostname and port
*/
public StartupSettings(String id, URI uri, GossipSettings gossipSettings, String cluster) {
this.id = id;
@ -84,30 +81,27 @@ public class StartupSettings {
/**
* Parse the settings for the gossip service from a JSON file.
*
* @param jsonFile
* The file object which refers to the JSON config file.
* @param jsonFile The file object which refers to the JSON config file.
* @return The StartupSettings object with the settings from the config file.
* @throws FileNotFoundException
* Thrown when the file cannot be found.
* @throws IOException
* Thrown when reading the file gives problems.
* @throws FileNotFoundException Thrown when the file cannot be found.
* @throws IOException Thrown when reading the file gives problems.
* @throws URISyntaxException
*/
public static StartupSettings fromJSONFile(File jsonFile) throws
FileNotFoundException, IOException, URISyntaxException {
public static StartupSettings fromJSONFile(File jsonFile)
throws FileNotFoundException, IOException, URISyntaxException {
ObjectMapper om = new ObjectMapper();
JsonNode root = om.readTree(jsonFile);
JsonNode jsonObject = root.get(0);
String uri = jsonObject.get("uri").textValue();
String id = jsonObject.get("id").textValue();
Map<String,String> properties = new HashMap<String,String>();
Map<String, String> properties = new HashMap<String, String>();
JsonNode n = jsonObject.get("properties");
Iterator<Entry<String, JsonNode>> l = n.fields();
while (l.hasNext()){
while (l.hasNext()) {
Entry<String, JsonNode> i = l.next();
properties.put(i.getKey(), i.getValue().asText());
}
//TODO constants as defaults?
// TODO constants as defaults?
// TODO setting keys as constants?
int gossipInterval = jsonObject.get("gossip_interval").intValue();
int cleanupInterval = jsonObject.get("cleanup_interval").intValue();
@ -116,24 +110,35 @@ public class StartupSettings {
double convictThreshold = jsonObject.get("convict_threshold").asDouble();
String cluster = jsonObject.get("cluster").textValue();
String distribution = jsonObject.get("distribution").textValue();
boolean bulkTransfer = jsonObject.has("bulk_transfer") ?
jsonObject.get("bulk_transfer").booleanValue() :
DEFAULT_BULK_TRANSFER;
int bulkTransferSize = jsonObject.has("bulk_transfer_size") ?
jsonObject.get("bulk_transfer_size").intValue() :
DEFAULT_BULK_TRANSFER_SIZE;
if (cluster == null){
boolean bulkTransfer =
jsonObject.has("bulk_transfer")
? jsonObject.get("bulk_transfer").booleanValue()
: DEFAULT_BULK_TRANSFER;
int bulkTransferSize =
jsonObject.has("bulk_transfer_size")
? jsonObject.get("bulk_transfer_size").intValue()
: DEFAULT_BULK_TRANSFER_SIZE;
if (cluster == null) {
throw new IllegalArgumentException("cluster was null. It is required");
}
String transportClass = jsonObject.has("transport_manager_class") ?
jsonObject.get("transport_manager_class").textValue() :
null;
String protocolClass = jsonObject.has("protocol_manager_class") ?
jsonObject.get("protocol_manager_class").textValue() :
null;
String transportClass =
jsonObject.has("transport_manager_class")
? jsonObject.get("transport_manager_class").textValue()
: null;
String protocolClass =
jsonObject.has("protocol_manager_class")
? jsonObject.get("protocol_manager_class").textValue()
: null;
URI uri2 = new URI(uri);
GossipSettings gossipSettings = new GossipSettings(gossipInterval, cleanupInterval, windowSize,
minSamples, convictThreshold, distribution, bulkTransfer);
GossipSettings gossipSettings =
new GossipSettings(
gossipInterval,
cleanupInterval,
windowSize,
minSamples,
convictThreshold,
distribution,
bulkTransfer);
gossipSettings.setBulkTransferSize(bulkTransferSize);
if (transportClass != null) {
gossipSettings.setTransportManagerClass(transportClass);
@ -146,9 +151,9 @@ public class StartupSettings {
JsonNode membersJSON = jsonObject.get("members");
for (JsonNode child : membersJSON) {
URI uri3 = new URI(child.get("uri").textValue());
RemoteMember member = new RemoteMember(child.get("cluster").asText(),
uri3, "", 0, new HashMap<String, String>()
);
RemoteMember member =
new RemoteMember(
child.get("cluster").asText(), uri3, "", 0, new HashMap<String, String>());
settings.addGossipMember(member);
configMembersDetails += member.computeAddress();
configMembersDetails += ", ";
@ -185,8 +190,7 @@ public class StartupSettings {
/**
* Set the id to be used for this service.
*
* @param id
* The id for this service.
* @param id The id for this service.
*/
public void setId(String id) {
this.id = id;
@ -204,8 +208,7 @@ public class StartupSettings {
/**
* Add a gossip member to the list of members to start with.
*
* @param member
* The member to add.
* @param member The member to add.
*/
public void addGossipMember(Member member) {
gossipMembers.add(member);

11
gossip-base/src/main/java/org/apache/gossip/accrual/FailureDetector.java
View File

@ -38,8 +38,7 @@ public class FailureDetector {
}
/**
* Updates the statistics based on the delta between the last
* heartbeat and supplied time
* Updates the statistics based on the delta between the last heartbeat and supplied time
*
* @param now the time of the heartbeat in milliseconds
*/
@ -63,9 +62,13 @@ public class FailureDetector {
if (distribution.equals("normal")) {
double standardDeviation = descriptiveStatistics.getStandardDeviation();
standardDeviation = Math.max(standardDeviation, 0.1);
probability = new NormalDistribution(descriptiveStatistics.getMean(), standardDeviation).cumulativeProbability(delta);
probability =
new NormalDistribution(descriptiveStatistics.getMean(), standardDeviation)
.cumulativeProbability(delta);
} else {
probability = new ExponentialDistribution(descriptiveStatistics.getMean()).cumulativeProbability(delta);
probability =
new ExponentialDistribution(descriptiveStatistics.getMean())
.cumulativeProbability(delta);
}
final double eps = 1e-12;
if (1 - probability < eps) {

116
gossip-base/src/main/java/org/apache/gossip/crdt/CrdtModule.java
View File

@ -37,76 +37,108 @@ import org.apache.gossip.replication.WhiteListReplicable;
abstract class OrSetMixin<E> {
@JsonCreator
OrSetMixin(@JsonProperty("elements") Map<E, Set<UUID>> w, @JsonProperty("tombstones") Map<E, Set<UUID>> h) { }
@JsonProperty("elements") abstract Map<E, Set<UUID>> getElements();
@JsonProperty("tombstones") abstract Map<E, Set<UUID>> getTombstones();
@JsonIgnore abstract boolean isEmpty();
OrSetMixin(
@JsonProperty("elements") Map<E, Set<UUID>> w,
@JsonProperty("tombstones") Map<E, Set<UUID>> h) {}
@JsonProperty("elements")
abstract Map<E, Set<UUID>> getElements();
@JsonProperty("tombstones")
abstract Map<E, Set<UUID>> getTombstones();
@JsonIgnore
abstract boolean isEmpty();
}
abstract class LWWSetMixin<ElementType> {
@JsonCreator
LWWSetMixin(@JsonProperty("data") Map<ElementType, LwwSet.Timestamps> struct) { }
@JsonProperty("data") abstract Map<ElementType, LwwSet.Timestamps> getStruct();
LWWSetMixin(@JsonProperty("data") Map<ElementType, LwwSet.Timestamps> struct) {}
@JsonProperty("data")
abstract Map<ElementType, LwwSet.Timestamps> getStruct();
}
abstract class LWWSetTimestampsMixin {
@JsonCreator
LWWSetTimestampsMixin(@JsonProperty("add") long latestAdd, @JsonProperty("remove") long latestRemove) { }
@JsonProperty("add") abstract long getLatestAdd();
@JsonProperty("remove") abstract long getLatestRemove();
LWWSetTimestampsMixin(
@JsonProperty("add") long latestAdd, @JsonProperty("remove") long latestRemove) {}
@JsonProperty("add")
abstract long getLatestAdd();
@JsonProperty("remove")
abstract long getLatestRemove();
}
abstract class MaxChangeSetMixin<E> {
@JsonCreator
MaxChangeSetMixin(@JsonProperty("data") Map<E, Integer> struct) { }
@JsonProperty("data") abstract Map<E, Integer> getStruct();
MaxChangeSetMixin(@JsonProperty("data") Map<E, Integer> struct) {}
@JsonProperty("data")
abstract Map<E, Integer> getStruct();
}
abstract class TwoPhaseSetMixin<E> {
@JsonCreator
TwoPhaseSetMixin(@JsonProperty("added") Set<E> added, @JsonProperty("removed") Set<E> removed) { }
@JsonProperty("added") abstract Set<E> getAdded();
@JsonProperty("removed") abstract Set<E> getRemoved();
TwoPhaseSetMixin(@JsonProperty("added") Set<E> added, @JsonProperty("removed") Set<E> removed) {}
@JsonProperty("added")
abstract Set<E> getAdded();
@JsonProperty("removed")
abstract Set<E> getRemoved();
}
abstract class GrowOnlySetMixin<E>{
abstract class GrowOnlySetMixin<E> {
@JsonCreator
GrowOnlySetMixin(@JsonProperty("elements") Set<E> elements){ }
@JsonProperty("elements") abstract Set<E> getElements();
@JsonIgnore abstract boolean isEmpty();
GrowOnlySetMixin(@JsonProperty("elements") Set<E> elements) {}
@JsonProperty("elements")
abstract Set<E> getElements();
@JsonIgnore
abstract boolean isEmpty();
}
abstract class GrowOnlyCounterMixin {
@JsonCreator
GrowOnlyCounterMixin(@JsonProperty("counters") Map<String, Long> counters) { }
@JsonProperty("counters") abstract Map<String, Long> getCounters();
GrowOnlyCounterMixin(@JsonProperty("counters") Map<String, Long> counters) {}
@JsonProperty("counters")
abstract Map<String, Long> getCounters();
}
abstract class PNCounterMixin {
@JsonCreator
PNCounterMixin(@JsonProperty("p-counters") Map<String, Long> up, @JsonProperty("n-counters") Map<String,Long> down) { }
@JsonProperty("p-counters") abstract Map<String, Long> getPCounters();
@JsonProperty("n-counters") abstract Map<String, Long> getNCounters();
PNCounterMixin(
@JsonProperty("p-counters") Map<String, Long> up,
@JsonProperty("n-counters") Map<String, Long> down) {}
@JsonProperty("p-counters")
abstract Map<String, Long> getPCounters();
@JsonProperty("n-counters")
abstract Map<String, Long> getNCounters();
}
@JsonTypeInfo(
use = JsonTypeInfo.Id.CLASS,
include = JsonTypeInfo.As.PROPERTY,
property = "type")
abstract class ReplicableMixin {
}
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "type")
abstract class ReplicableMixin {}
abstract class WhiteListReplicableMixin {
@JsonCreator
WhiteListReplicableMixin(@JsonProperty("whiteListMembers") List<LocalMember> whiteListMembers) { }
@JsonProperty("whiteListMembers") abstract List<LocalMember> getWhiteListMembers();
WhiteListReplicableMixin(@JsonProperty("whiteListMembers") List<LocalMember> whiteListMembers) {}
@JsonProperty("whiteListMembers")
abstract List<LocalMember> getWhiteListMembers();
}
abstract class BlackListReplicableMixin {
@JsonCreator
BlackListReplicableMixin(@JsonProperty("blackListMembers") List<LocalMember> blackListMembers) { }
@JsonProperty("blackListMembers") abstract List<LocalMember> getBlackListMembers();
BlackListReplicableMixin(@JsonProperty("blackListMembers") List<LocalMember> blackListMembers) {}
@JsonProperty("blackListMembers")
abstract List<LocalMember> getBlackListMembers();
}
abstract class VoteCandidateMixin {
@ -114,8 +146,7 @@ abstract class VoteCandidateMixin {
VoteCandidateMixin(
@JsonProperty("candidateNodeId") String candidateNodeId,
@JsonProperty("votingKey") String votingKey,
@JsonProperty("votes") Map<String, Vote> votes
) { }
@JsonProperty("votes") Map<String, Vote> votes) {}
}
abstract class VoteMixin {
@ -125,17 +156,16 @@ abstract class VoteMixin {
@JsonProperty("voteValue") Boolean voteValue,
@JsonProperty("voteExchange") Boolean voteExchange,
@JsonProperty("liveMembers") List<String> liveMembers,
@JsonProperty("deadMembers") List<String> deadMembers
) { }
@JsonProperty("deadMembers") List<String> deadMembers) {}
}
abstract class MajorityVoteMixin<E>{
abstract class MajorityVoteMixin<E> {
@JsonCreator
MajorityVoteMixin(@JsonProperty("voteCandidates") Map<String, VoteCandidate> voteCandidateMap){ }
MajorityVoteMixin(@JsonProperty("voteCandidates") Map<String, VoteCandidate> voteCandidateMap) {}
}
//If anyone wants to take a stab at this. please have at it
//https://github.com/FasterXML/jackson-datatype-guava/blob/master/src/main/java/com/fasterxml/jackson/datatype/guava/ser/MultimapSerializer.java
// If anyone wants to take a stab at this. please have at it
// https://github.com/FasterXML/jackson-datatype-guava/blob/master/src/main/java/com/fasterxml/jackson/datatype/guava/ser/MultimapSerializer.java
public class CrdtModule extends SimpleModule {
private static final long serialVersionUID = 6134836523275023418L;
@ -161,6 +191,4 @@ public class CrdtModule extends SimpleModule {
context.setMixInAnnotations(VoteCandidate.class, VoteCandidateMixin.class);
context.setMixInAnnotations(Vote.class, VoteMixin.class);
}
}

3
gossip-base/src/main/java/org/apache/gossip/crdt/GrowOnlyCounter.java
View File

@ -84,8 +84,7 @@ public class GrowOnlyCounter implements CrdtCounter<Long, GrowOnlyCounter> {
@Override
public boolean equals(Object obj) {
if (getClass() != obj.getClass())
return false;
if (getClass() != obj.getClass()) return false;
GrowOnlyCounter other = (GrowOnlyCounter) obj;
return value().longValue() == other.value().longValue();
}

69
gossip-base/src/main/java/org/apache/gossip/crdt/LwwSet.java
View File

@ -43,57 +43,60 @@ import org.apache.gossip.manager.SystemClock;
DataTest - integration test with 2 nodes, LWWSet was serialized/deserialized, sent between nodes, merged
*/
public class LwwSet<ElementType> implements CrdtAddRemoveSet<ElementType, Set<ElementType>, LwwSet<ElementType>> {
static private Clock clock = new SystemClock();
public class LwwSet<ElementType>
implements CrdtAddRemoveSet<ElementType, Set<ElementType>, LwwSet<ElementType>> {
private static Clock clock = new SystemClock();
private final Map<ElementType, Timestamps> struct;
public LwwSet(){
public LwwSet() {
struct = new HashMap<>();
}
@SafeVarargs
public LwwSet(ElementType... elements){
public LwwSet(ElementType... elements) {
this(new HashSet<>(Arrays.asList(elements)));
}
public LwwSet(Set<ElementType> set){
public LwwSet(Set<ElementType> set) {
struct = new HashMap<>();
for (ElementType e : set){
for (ElementType e : set) {
struct.put(e, new Timestamps().updateAdd());
}
}
public LwwSet(LwwSet<ElementType> first, LwwSet<ElementType> second){
Function<ElementType, Timestamps> timestampsFor = p -> {
public LwwSet(LwwSet<ElementType> first, LwwSet<ElementType> second) {
Function<ElementType, Timestamps> timestampsFor =
p -> {
Timestamps firstTs = first.struct.get(p);
Timestamps secondTs = second.struct.get(p);
if (firstTs == null){
if (firstTs == null) {
return secondTs;
}
return firstTs.merge(secondTs);
};
struct = Stream.concat(first.struct.keySet().stream(), second.struct.keySet().stream())
.distinct().collect(Collectors.toMap(p -> p, timestampsFor));
struct =
Stream.concat(first.struct.keySet().stream(), second.struct.keySet().stream())
.distinct()
.collect(Collectors.toMap(p -> p, timestampsFor));
}
// for serialization
LwwSet(Map<ElementType, Timestamps> struct){
LwwSet(Map<ElementType, Timestamps> struct) {
this.struct = struct;
}
public LwwSet<ElementType> add(ElementType e){
public LwwSet<ElementType> add(ElementType e) {
return this.merge(new LwwSet<>(e));
}
Map<ElementType, Timestamps> getStruct(){
Map<ElementType, Timestamps> getStruct() {
return struct;
}
public LwwSet<ElementType> remove(ElementType e){
public LwwSet<ElementType> remove(ElementType e) {
Timestamps eTimestamps = struct.get(e);
if (eTimestamps == null || !eTimestamps.isPresent()){
if (eTimestamps == null || !eTimestamps.isPresent()) {
return this;
}
Map<ElementType, Timestamps> changeMap = new HashMap<>();
@ -102,12 +105,12 @@ public class LwwSet<ElementType> implements CrdtAddRemoveSet<ElementType, Set<El
}
@Override
public LwwSet<ElementType> merge(LwwSet<ElementType> other){
public LwwSet<ElementType> merge(LwwSet<ElementType> other) {
return new LwwSet<>(this, other);
}
@Override
public Set<ElementType> value(){
public Set<ElementType> value() {
return struct.entrySet().stream()
.filter(entry -> entry.getValue().isPresent())
.map(Map.Entry::getKey)
@ -115,55 +118,57 @@ public class LwwSet<ElementType> implements CrdtAddRemoveSet<ElementType, Set<El
}
@Override
public LwwSet<ElementType> optimize(){
public LwwSet<ElementType> optimize() {
return this;
}
@Override
public boolean equals(Object obj){
return this == obj || (obj != null && getClass() == obj.getClass() && value().equals(((LwwSet) obj).value()));
public boolean equals(Object obj) {
return this == obj
|| (obj != null && getClass() == obj.getClass() && value().equals(((LwwSet) obj).value()));
}
static class Timestamps {
private final long latestAdd;
private final long latestRemove;
Timestamps(){
Timestamps() {
latestAdd = 0;
latestRemove = 0;
}
Timestamps(long add, long remove){
Timestamps(long add, long remove) {
latestAdd = add;
latestRemove = remove;
}
long getLatestAdd(){
long getLatestAdd() {
return latestAdd;
}
long getLatestRemove(){
long getLatestRemove() {
return latestRemove;
}
// consider element present when addTime >= removeTime, so we prefer add to remove
boolean isPresent(){
boolean isPresent() {
return latestAdd >= latestRemove;
}
Timestamps updateAdd(){
Timestamps updateAdd() {
return new Timestamps(clock.nanoTime(), latestRemove);
}
Timestamps updateRemove(){
Timestamps updateRemove() {
return new Timestamps(latestAdd, clock.nanoTime());
}
Timestamps merge(Timestamps other){
if (other == null){
Timestamps merge(Timestamps other) {
if (other == null) {
return this;
}
return new Timestamps(Math.max(latestAdd, other.latestAdd), Math.max(latestRemove, other.latestRemove));
return new Timestamps(
Math.max(latestAdd, other.latestAdd), Math.max(latestRemove, other.latestRemove));
}
}
}

63
gossip-base/src/main/java/org/apache/gossip/crdt/OrSet.java
View File

@ -31,31 +31,31 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
private final Map<E, Set<UUID>> tombstones = new HashMap<>();
private final transient Set<E> val;
public OrSet(){
public OrSet() {
val = computeValue();
}
OrSet(Map<E, Set<UUID>> elements, Map<E, Set<UUID>> tombstones){
OrSet(Map<E, Set<UUID>> elements, Map<E, Set<UUID>> tombstones) {
this.elements.putAll(elements);
this.tombstones.putAll(tombstones);
val = computeValue();
}
@SafeVarargs
public OrSet(E ... elements){
public OrSet(E... elements) {
this(new HashSet<>(Arrays.asList(elements)));
}
public OrSet(Set<E> elements) {
for (E e: elements){
for (E e : elements) {
internalAdd(e);
}
val = computeValue();
}
public OrSet(Builder<E>builder){
for (Builder<E>.OrSetElement<E> e: builder.elements){
if (e.operation == Operation.ADD){
public OrSet(Builder<E> builder) {
for (Builder<E>.OrSetElement<E> e : builder.elements) {
if (e.operation == Operation.ADD) {
internalAdd(e.element);
} else {
internalRemove(e.element);
@ -66,14 +66,15 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
/**
* This constructor is the way to remove elements from an existing set
*
* @param set
* @param builder
*/
public OrSet(OrSet<E> set, Builder<E> builder){
public OrSet(OrSet<E> set, Builder<E> builder) {
elements.putAll(set.elements);
tombstones.putAll(set.tombstones);
for (Builder<E>.OrSetElement<E> e: builder.elements){
if (e.operation == Operation.ADD){
for (Builder<E>.OrSetElement<E> e : builder.elements) {
if (e.operation == Operation.ADD) {
internalAdd(e.element);
} else {
internalRemove(e.element);
@ -82,9 +83,10 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
val = computeValue();
}
public OrSet(OrSet<E> left, OrSet<E> right){
BiConsumer<Map<E, Set<UUID>>, Map<E, Set<UUID>>> internalMerge = (items, other) -> {
for (Entry<E, Set<UUID>> l : other.entrySet()){
public OrSet(OrSet<E> left, OrSet<E> right) {
BiConsumer<Map<E, Set<UUID>>, Map<E, Set<UUID>>> internalMerge =
(items, other) -> {
for (Entry<E, Set<UUID>> l : other.entrySet()) {
internalSetMerge(items, l.getKey(), l.getValue());
}
};
@ -121,7 +123,7 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
return new OrSet<>(this, new Builder<E>().remove(e));
}
public OrSet.Builder<E> builder(){
public OrSet.Builder<E> builder() {
return new OrSet.Builder<>();
}
@ -136,16 +138,16 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
internalSetMerge(elements, element, toMerge);
}
private void internalRemove(E element){
private void internalRemove(E element) {
internalSetMerge(tombstones, element, elements.get(element));
}
/*
* Computes the live values by analyzing the elements and tombstones
*/
private Set<E> computeValue(){
private Set<E> computeValue() {
Set<E> values = new HashSet<>();
for (Entry<E, Set<UUID>> entry: elements.entrySet()){
for (Entry<E, Set<UUID>> entry : elements.entrySet()) {
Set<UUID> deleteIds = tombstones.get(entry.getKey());
// if not all tokens for current element are in tombstones
if (deleteIds == null || !deleteIds.containsAll(entry.getValue())) {
@ -195,7 +197,6 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
public E next() {
return managed.next();
}
};
}
@ -229,7 +230,7 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
@Override
public String toString() {
return "OrSet [elements=" + elements + ", tombstones=" + tombstones + "]" ;
return "OrSet [elements=" + elements + ", tombstones=" + tombstones + "]";
}
@Override
@ -242,19 +243,14 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
@SuppressWarnings("rawtypes")
OrSet other = (OrSet) obj;
if (elements == null) {
if (other.elements != null)
return false;
} else if (!value().equals(other.value()))
return false;
if (other.elements != null) return false;
} else if (!value().equals(other.value())) return false;
return true;
}
@ -267,7 +263,8 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
}
public static class Builder<E> {
private List<OrSetElement<E>> elements = new ArrayList<>();;
private List<OrSetElement<E>> elements = new ArrayList<>();
;
public Builder<E> add(E element) {
elements.add(new OrSetElement<E>(element, Operation.ADD));
@ -284,8 +281,9 @@ public class OrSet<E> implements CrdtAddRemoveSet<E, Set<E>, OrSet<E>> {
return this;
}
public static enum Operation {
ADD, REMOVE
public static enum Operation {
ADD,
REMOVE
}
private class OrSetElement<EL> {
@ -298,5 +296,4 @@ public static enum Operation {
}
}
}
}

8
gossip-base/src/main/java/org/apache/gossip/crdt/PNCounter.java
View File

@ -74,8 +74,7 @@ public class PNCounter implements CrdtCounter<Long, PNCounter> {
@Override
public boolean equals(Object obj) {
if (getClass() != obj.getClass())
return false;
if (getClass() != obj.getClass()) return false;
PNCounter other = (PNCounter) obj;
return value().longValue() == other.value().longValue();
}
@ -118,8 +117,8 @@ public class PNCounter implements CrdtCounter<Long, PNCounter> {
}
public org.apache.gossip.crdt.GrowOnlyCounter.Builder makeGrowOnlyCounterBuilder(long value) {
org.apache.gossip.crdt.GrowOnlyCounter.Builder ret = new org.apache.gossip.crdt.GrowOnlyCounter.Builder(
myManager);
org.apache.gossip.crdt.GrowOnlyCounter.Builder ret =
new org.apache.gossip.crdt.GrowOnlyCounter.Builder(myManager);
ret.increment(value);
return ret;
}
@ -134,5 +133,4 @@ public class PNCounter implements CrdtCounter<Long, PNCounter> {
return this;
}
}
}

1
gossip-base/src/main/java/org/apache/gossip/event/data/DataEventManager.java
View File

@ -19,7 +19,6 @@ package org.apache.gossip.event.data;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;

89
gossip-base/src/main/java/org/apache/gossip/lock/LockManager.java
View File

@ -46,7 +46,6 @@ import org.apache.gossip.model.SharedDataMessage;
@Slf4j
public class LockManager {
// For MetricRegistry
public static final String LOCK_KEY_SET_SIZE = "gossip.lock.key_set_size";
public static final String LOCK_TIME = "gossip.lock.time";
@ -57,7 +56,9 @@ public class LockManager {
private final Set<String> lockKeys;
private final Timer lockTimeMetric;
public LockManager(GossipManager gossipManager, final LockManagerSettings lockManagerSettings,
public LockManager(
GossipManager gossipManager,
final LockManagerSettings lockManagerSettings,
MetricRegistry metrics) {
this.gossipManager = gossipManager;
this.lockSettings = lockManagerSettings;
@ -66,14 +67,15 @@ public class LockManager {
metrics.register(LOCK_KEY_SET_SIZE, (Gauge<Integer>) lockKeys::size);
lockTimeMetric = metrics.timer(LOCK_TIME);
// Register listener for lock keys
gossipManager.registerSharedDataSubscriber((key, oldValue, newValue) -> {
gossipManager.registerSharedDataSubscriber(
(key, oldValue, newValue) -> {
if (key.contains("lock/")) {
lockKeys.add(key);
}
});
voteService = Executors.newScheduledThreadPool(2);
voteService.scheduleAtFixedRate(this::updateVotes, 0, lockSettings.getVoteUpdateInterval(),
TimeUnit.MILLISECONDS);
voteService.scheduleAtFixedRate(
this::updateVotes, 0, lockSettings.getVoteUpdateInterval(), TimeUnit.MILLISECONDS);
}
public void acquireSharedDataLock(String key) throws VoteFailedException {
@ -89,12 +91,18 @@ public class LockManager {
final Map<String, VoteCandidate> voteCandidatesMap = majorityVoteResult.value();
final Map<String, Boolean> voteResultMap = new HashMap<>();
// Store the vote result for each vote candidate nodes
voteCandidatesMap.forEach((candidateId, voteCandidate) -> voteResultMap
.put(candidateId, isVoteSuccess(voteCandidate)));
voteCandidatesMap.forEach(
(candidateId, voteCandidate) ->
voteResultMap.put(candidateId, isVoteSuccess(voteCandidate)));
long passedCandidates = voteResultMap.values().stream().filter(aBoolean -> aBoolean).count();
String myNodeId = gossipManager.getMyself().getId();
log.debug("NodeId=" + myNodeId + ", VoteMap=" + voteResultMap + ", WinnerCount="
log.debug(
"NodeId="
+ myNodeId
+ ", VoteMap="
+ voteResultMap
+ ", WinnerCount="
+ passedCandidates);
// Check for possible dead lock when no candidates were won
if (passedCandidates == 0) {
@ -102,7 +110,10 @@ public class LockManager {
deadlockDetectCount++;
// Testing for deadlock is not always correct, therefore test for continues deadlocks
if (deadlockDetectCount >= lockSettings.getDeadlockDetectionThreshold()) {
log.debug("Deadlock detected from node " + myNodeId + ". VoteCandidatesMap="
log.debug(
"Deadlock detected from node "
+ myNodeId
+ ". VoteCandidatesMap="
+ voteCandidatesMap);
preventDeadLock(voteCandidatesMap);
}
@ -133,11 +144,16 @@ public class LockManager {
// Generate Crdt lock message for voting
private SharedDataMessage generateLockMessage(String key) {
VoteCandidate voteCandidate = new VoteCandidate(gossipManager.getMyself().getId(), key,
new ConcurrentHashMap<>());
voteCandidate.addVote(new Vote(gossipManager.getMyself().getId(), true, false,
VoteCandidate voteCandidate =
new VoteCandidate(gossipManager.getMyself().getId(), key, new ConcurrentHashMap<>());
voteCandidate.addVote(
new Vote(
gossipManager.getMyself().getId(),
true,
false,
gossipManager.getLiveMembers().stream().map(Member::getId).collect(Collectors.toList()),
gossipManager.getDeadMembers().stream().map(Member::getId)
gossipManager.getDeadMembers().stream()
.map(Member::getId)
.collect(Collectors.toList())));
Map<String, VoteCandidate> voteCandidateMap = new ConcurrentHashMap<>();
voteCandidateMap.put(voteCandidate.getCandidateNodeId(), voteCandidate);
@ -167,7 +183,8 @@ public class LockManager {
String myVoteCandidate = getVotedCandidateNodeId(myNodeId, voteCandidateMap);
if (myVoteCandidate == null) {
myVoteCandidate = lockSettings.getVoteSelector().getVoteCandidateId(voteCandidateMap.keySet());
myVoteCandidate =
lockSettings.getVoteSelector().getVoteCandidateId(voteCandidateMap.keySet());
}
for (VoteCandidate voteCandidate : voteCandidateMap.values()) {
if (voteCandidate.getCandidateNodeId().equals(myNodeId)) {
@ -175,10 +192,16 @@ public class LockManager {
}
// Vote for selected candidate
boolean voteResult = voteCandidate.getCandidateNodeId().equals(myVoteCandidate);
voteCandidate.addVote(new Vote(gossipManager.getMyself().getId(), voteResult, false,
gossipManager.getLiveMembers().stream().map(Member::getId)
voteCandidate.addVote(
new Vote(
gossipManager.getMyself().getId(),
voteResult,
false,
gossipManager.getLiveMembers().stream()
.map(Member::getId)
.collect(Collectors.toList()),
gossipManager.getDeadMembers().stream().map(Member::getId)
gossipManager.getDeadMembers().stream()
.map(Member::getId)
.collect(Collectors.toList())));
}
}
@ -203,9 +226,13 @@ public class LockManager {
numberOfLiveNodes = numberOfNodes.get();
} else {
// numberOfNodes is not set by the user, therefore calculate it.
Set<String> liveNodes = voteCandidates.values().stream()
.map(voteCandidate -> voteCandidate.getVotes().values()).flatMap(Collection::stream)
.map(Vote::getLiveMembers).flatMap(List::stream).collect(Collectors.toSet());
Set<String> liveNodes =
voteCandidates.values().stream()
.map(voteCandidate -> voteCandidate.getVotes().values())
.flatMap(Collection::stream)
.map(Vote::getLiveMembers)
.flatMap(List::stream)
.collect(Collectors.toSet());
numberOfLiveNodes = liveNodes.size();
}
for (VoteCandidate voteCandidate : voteCandidates.values()) {
@ -222,15 +249,17 @@ public class LockManager {
return;
}
// Set of nodes that is going to receive this nodes votes
List<String> donateCandidateIds = voteCandidates.keySet().stream()
.filter(s -> s.compareTo(myNodeId) < 0).collect(Collectors.toList());
List<String> donateCandidateIds =
voteCandidates.keySet().stream()
.filter(s -> s.compareTo(myNodeId) < 0)
.collect(Collectors.toList());
if (donateCandidateIds.size() == 0) {
return;
}
// Select a random node to donate
Random randomizer = new Random();
String selectedCandidateId = donateCandidateIds
.get(randomizer.nextInt(donateCandidateIds.size()));
String selectedCandidateId =
donateCandidateIds.get(randomizer.nextInt(donateCandidateIds.size()));
VoteCandidate selectedCandidate = voteCandidates.get(selectedCandidateId);
Set<Vote> myVotes = new HashSet<>(myResults.getVotes().values());
@ -249,8 +278,8 @@ public class LockManager {
log.debug("Node " + myNodeId + " give up votes to node " + selectedCandidateId);
}
private String getVotedCandidateNodeId(String nodeId,
final Map<String, VoteCandidate> voteCandidates) {
private String getVotedCandidateNodeId(
String nodeId, final Map<String, VoteCandidate> voteCandidates) {
for (VoteCandidate voteCandidate : voteCandidates.values()) {
Vote vote = voteCandidate.getVotes().get(nodeId);
if (vote != null && vote.getVoteValue()) {
@ -279,15 +308,17 @@ public class LockManager {
return numberOfLiveNodes > 0 && voteCount >= (numberOfLiveNodes / 2 + 1);
}
private String generateLockKey(String key){
private String generateLockKey(String key) {
return "lock/" + key;
}
public void shutdown(){
public void shutdown() {
voteService.shutdown();
}
/**
* Get the voted node id from this node for a given key
*
* @param key key of the data object
* @return Voted node id
*/
@ -302,10 +333,10 @@ public class LockManager {
/**
* Set the number of live nodes. If this value is negative, live nodes will be calculated
*
* @param numberOfNodes live node count or negative to calculate.
*/
public void setNumberOfNodes(int numberOfNodes) {
this.numberOfNodes.set(numberOfNodes);
}
}

54
gossip-base/src/main/java/org/apache/gossip/lock/vote/MajorityVote.java
View File

@ -25,9 +25,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.gossip.crdt.Crdt;
/**
* CRDT which used for distribute a votes for a given key.
*/
/** CRDT which used for distribute a votes for a given key. */
public class MajorityVote implements Crdt<Map<String, VoteCandidate>, MajorityVote> {
private final Map<String, VoteCandidate> voteCandidates = new ConcurrentHashMap<>();
@ -53,10 +51,12 @@ public class MajorityVote implements Crdt<Map<String, VoteCandidate>, MajorityVo
}
// Merge votes for the same candidate
for (String sameCandidateId : sameCandidatesSet) {
if (this.voteCandidates.containsKey(sameCandidateId) && other.voteCandidates
.containsKey(sameCandidateId)) {
mergedCandidates.put(sameCandidateId,
mergeCandidate(this.voteCandidates.get(sameCandidateId),
if (this.voteCandidates.containsKey(sameCandidateId)
&& other.voteCandidates.containsKey(sameCandidateId)) {
mergedCandidates.put(
sameCandidateId,
mergeCandidate(
this.voteCandidates.get(sameCandidateId),
other.voteCandidates.get(sameCandidateId)));
}
}
@ -65,10 +65,13 @@ public class MajorityVote implements Crdt<Map<String, VoteCandidate>, MajorityVo
}
// Merge different votes for same candidate
private VoteCandidate mergeCandidate(VoteCandidate firstCandidate,
VoteCandidate secondCandidate) {
VoteCandidate mergeResult = new VoteCandidate(firstCandidate.getCandidateNodeId(),
firstCandidate.getVotingKey(), new ConcurrentHashMap<>());
private VoteCandidate mergeCandidate(
VoteCandidate firstCandidate, VoteCandidate secondCandidate) {
VoteCandidate mergeResult =
new VoteCandidate(
firstCandidate.getCandidateNodeId(),
firstCandidate.getVotingKey(),
new ConcurrentHashMap<>());
Set<String> firstKeySet = firstCandidate.getVotes().keySet();
Set<String> secondKeySet = secondCandidate.getVotes().keySet();
Set<String> sameVoteNodeSet = getIntersection(firstKeySet, secondKeySet);
@ -76,19 +79,25 @@ public class MajorityVote implements Crdt<Map<String, VoteCandidate>, MajorityVo
// Merge different voters by combining their votes
for (String differentCandidateId : differentVoteNodeSet) {
if (firstCandidate.getVotes().containsKey(differentCandidateId)) {
mergeResult.getVotes()
mergeResult
.getVotes()
.put(differentCandidateId, firstCandidate.getVotes().get(differentCandidateId));
} else if (secondCandidate.getVotes().containsKey(differentCandidateId)) {
mergeResult.getVotes()
mergeResult
.getVotes()
.put(differentCandidateId, secondCandidate.getVotes().get(differentCandidateId));
}
}
// Merge vote for same voter
for (String sameVoteNodeId : sameVoteNodeSet) {
if (firstCandidate.getVotes().containsKey(sameVoteNodeId) && secondCandidate.getVotes()
.containsKey(sameVoteNodeId)) {
mergeResult.getVotes().put(sameVoteNodeId,
mergeVote(firstCandidate.getVotes().get(sameVoteNodeId),
if (firstCandidate.getVotes().containsKey(sameVoteNodeId)
&& secondCandidate.getVotes().containsKey(sameVoteNodeId)) {
mergeResult
.getVotes()
.put(
sameVoteNodeId,
mergeVote(
firstCandidate.getVotes().get(sameVoteNodeId),
secondCandidate.getVotes().get(sameVoteNodeId)));
}
}
@ -131,7 +140,6 @@ public class MajorityVote implements Crdt<Map<String, VoteCandidate>, MajorityVo
Map<String, VoteCandidate> copy = new ConcurrentHashMap<>();
copy.putAll(voteCandidates);
return Collections.unmodifiableMap(copy);
}
@Override
@ -141,12 +149,9 @@ public class MajorityVote implements Crdt<Map<String, VoteCandidate>, MajorityVo
@Override
public boolean equals(Object obj) {
if (obj == null)
return false;
if (obj == this)
return true;
if (!(obj instanceof MajorityVote))
return false;
if (obj == null) return false;
if (obj == this) return true;
if (!(obj instanceof MajorityVote)) return false;
MajorityVote other = (MajorityVote) obj;
return Objects.equals(voteCandidates, other.voteCandidates);
}
@ -164,5 +169,4 @@ public class MajorityVote implements Crdt<Map<String, VoteCandidate>, MajorityVo
public Map<String, VoteCandidate> getVoteCandidates() {
return new ConcurrentHashMap<>(voteCandidates);
}
}

20
gossip-base/src/main/java/org/apache/gossip/lock/vote/Vote.java
View File

@ -19,9 +19,7 @@ package org.apache.gossip.lock.vote;
import java.util.List;
/**
* Store a voter details.
*/
/** Store a voter details. */
public class Vote {
private final String votingNode;
private final Boolean voteValue; // TODO: 7/16/17 weight?
@ -29,7 +27,11 @@ public class Vote {
private final List<String> deadMembers;
private Boolean voteExchange;
public Vote(String votingNode, Boolean voteValue, Boolean voteExchange, List<String> liveMembers,
public Vote(
String votingNode,
Boolean voteValue,
Boolean voteExchange,
List<String> liveMembers,
List<String> deadMembers) {
this.votingNode = votingNode;
this.voteValue = voteValue;
@ -64,7 +66,13 @@ public class Vote {
@Override
public String toString() {
return "votingNode=" + votingNode + ", voteValue=" + voteValue + ", liveMembers=" + liveMembers
+ ", deadMembers= " + deadMembers;
return "votingNode="
+ votingNode
+ ", voteValue="
+ voteValue
+ ", liveMembers="
+ liveMembers
+ ", deadMembers= "
+ deadMembers;
}
}

80
gossip-base/src/main/java/org/apache/gossip/manager/AbstractActiveGossiper.java
View File

@ -38,7 +38,8 @@ import org.apache.gossip.model.ShutdownMessage;
import org.apache.gossip.udp.*;
/**
* The ActiveGossipThread sends information. Pick a random partner and send the membership list to that partner
* The ActiveGossipThread sends information. Pick a random partner and send the membership list to
* that partner
*/
@Slf4j
public abstract class AbstractActiveGossiper {
@ -51,26 +52,26 @@ public abstract class AbstractActiveGossiper {
private final Random random;
private final GossipSettings gossipSettings;
public AbstractActiveGossiper(GossipManager gossipManager, GossipCore gossipCore, MetricRegistry registry) {
public AbstractActiveGossiper(
GossipManager gossipManager, GossipCore gossipCore, MetricRegistry registry) {
this.gossipManager = gossipManager;
this.gossipCore = gossipCore;
sharedDataHistogram = registry.histogram(name(AbstractActiveGossiper.class, "sharedDataHistogram-time"));
sendPerNodeDataHistogram = registry.histogram(name(AbstractActiveGossiper.class, "sendPerNodeDataHistogram-time"));
sendMembershipHistogram = registry.histogram(name(AbstractActiveGossiper.class, "sendMembershipHistogram-time"));
sharedDataHistogram =
registry.histogram(name(AbstractActiveGossiper.class, "sharedDataHistogram-time"));
sendPerNodeDataHistogram =
registry.histogram(name(AbstractActiveGossiper.class, "sendPerNodeDataHistogram-time"));
sendMembershipHistogram =
registry.histogram(name(AbstractActiveGossiper.class, "sendMembershipHistogram-time"));
random = new Random();
gossipSettings = gossipManager.getSettings();
}
public void init() {
public void init() {}
}
public void shutdown() {}
public void shutdown() {
}
public final void sendShutdownMessage(LocalMember me, LocalMember target){
if (target == null){
public final void sendShutdownMessage(LocalMember me, LocalMember target) {
if (target == null) {
return;
}
ShutdownMessage m = new ShutdownMessage();
@ -94,8 +95,11 @@ public abstract class AbstractActiveGossiper {
/** Send shared data one entry at a time. */
private void sendSharedDataInternal(LocalMember me, LocalMember member) {
for (Entry<String, SharedDataMessage> innerEntry : gossipCore.getSharedData().entrySet()){
if (innerEntry.getValue().getReplicable() != null && !innerEntry.getValue().getReplicable()
for (Entry<String, SharedDataMessage> innerEntry : gossipCore.getSharedData().entrySet()) {
if (innerEntry.getValue().getReplicable() != null
&& !innerEntry
.getValue()
.getReplicable()
.shouldReplicate(me, member, innerEntry.getValue())) {
continue;
}
@ -113,7 +117,10 @@ public abstract class AbstractActiveGossiper {
udpMessage.setUuid(UUID.randomUUID().toString());
udpMessage.setUriFrom(me.getId());
for (Entry<String, SharedDataMessage> innerEntry : gossipCore.getSharedData().entrySet()) {
if (innerEntry.getValue().getReplicable() != null && !innerEntry.getValue().getReplicable()
if (innerEntry.getValue().getReplicable() != null
&& !innerEntry
.getValue()
.getReplicable()
.shouldReplicate(me, member, innerEntry.getValue())) {
continue;
}
@ -141,8 +148,8 @@ public abstract class AbstractActiveGossiper {
copy.setReplicable(original.getReplicable());
}
public final void sendPerNodeData(LocalMember me, LocalMember member){
if (member == null){
public final void sendPerNodeData(LocalMember me, LocalMember member) {
if (member == null) {
return;
}
long startTime = System.currentTimeMillis();
@ -156,9 +163,13 @@ public abstract class AbstractActiveGossiper {
/** Send per node data one entry at a time. */
private void sendPerNodeDataInternal(LocalMember me, LocalMember member) {
for (Entry<String, ConcurrentHashMap<String, PerNodeDataMessage>> entry : gossipCore.getPerNodeData().entrySet()){
for (Entry<String, PerNodeDataMessage> innerEntry : entry.getValue().entrySet()){
if (innerEntry.getValue().getReplicable() != null && !innerEntry.getValue().getReplicable()
for (Entry<String, ConcurrentHashMap<String, PerNodeDataMessage>> entry :
gossipCore.getPerNodeData().entrySet()) {
for (Entry<String, PerNodeDataMessage> innerEntry : entry.getValue().entrySet()) {
if (innerEntry.getValue().getReplicable() != null
&& !innerEntry
.getValue()
.getReplicable()
.shouldReplicate(me, member, innerEntry.getValue())) {
continue;
}
@ -169,17 +180,20 @@ public abstract class AbstractActiveGossiper {
gossipCore.sendOneWay(message, member.getUri());
}
}
}
/** Send per node data by batching together several entries. */
private void sendPerNodeDataInBulkInternal(LocalMember me, LocalMember member) {
for (Entry<String, ConcurrentHashMap<String, PerNodeDataMessage>> entry : gossipCore.getPerNodeData().entrySet()){
for (Entry<String, ConcurrentHashMap<String, PerNodeDataMessage>> entry :
gossipCore.getPerNodeData().entrySet()) {
UdpPerNodeDataBulkMessage udpMessage = new UdpPerNodeDataBulkMessage();
udpMessage.setUuid(UUID.randomUUID().toString());
udpMessage.setUriFrom(me.getId());
for (Entry<String, PerNodeDataMessage> innerEntry : entry.getValue().entrySet()){
if (innerEntry.getValue().getReplicable() != null && !innerEntry.getValue().getReplicable()
for (Entry<String, PerNodeDataMessage> innerEntry : entry.getValue().entrySet()) {
if (innerEntry.getValue().getReplicable() != null
&& !innerEntry
.getValue()
.getReplicable()
.shouldReplicate(me, member, innerEntry.getValue())) {
continue;
}
@ -208,11 +222,9 @@ public abstract class AbstractActiveGossiper {
copy.setReplicable(original.getReplicable());
}
/**
* Performs the sending of the membership list, after we have incremented our own heartbeat.
*/
/** Performs the sending of the membership list, after we have incremented our own heartbeat. */
protected void sendMembershipList(LocalMember me, LocalMember member) {
if (member == null){
if (member == null) {
return;
}
long startTime = System.currentTimeMillis();
@ -225,15 +237,15 @@ public abstract class AbstractActiveGossiper {
message.getMembers().add(convert(other));
}
Response r = gossipCore.send(message, member.getUri());
if (r instanceof ActiveGossipOk){
//maybe count metrics here
if (r instanceof ActiveGossipOk) {
// maybe count metrics here
} else {
log.debug("Message " + message + " generated response " + r);
}
sendMembershipHistogram.update(System.currentTimeMillis() - startTime);
}
protected final Member convert(LocalMember member){
protected final Member convert(LocalMember member) {
Member gm = new Member();
gm.setCluster(member.getClusterName());
gm.setHeartbeat(member.getHeartbeat());
@ -244,9 +256,7 @@ public abstract class AbstractActiveGossiper {
}
/**
*
* @param memberList
* An immutable list
* @param memberList An immutable list
* @return The chosen LocalGossipMember to gossip with.
*/
protected LocalMember selectPartner(List<LocalMember> memberList) {

26
gossip-base/src/main/java/org/apache/gossip/manager/DataReaper.java
View File

@ -38,42 +38,44 @@ public class DataReaper {
private final ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(1);
private final Clock clock;
public DataReaper(GossipCore gossipCore, Clock clock){
public DataReaper(GossipCore gossipCore, Clock clock) {
this.gossipCore = gossipCore;
this.clock = clock;
}
public void init(){
Runnable reapPerNodeData = () -> {
public void init() {
Runnable reapPerNodeData =
() -> {
runPerNodeOnce();
runSharedOnce();
};
scheduledExecutor.scheduleAtFixedRate(reapPerNodeData, 0, 5, TimeUnit.SECONDS);
}
void runSharedOnce(){
for (Entry<String, SharedDataMessage> entry : gossipCore.getSharedData().entrySet()){
if (entry.getValue().getExpireAt() < clock.currentTimeMillis()){
void runSharedOnce() {
for (Entry<String, SharedDataMessage> entry : gossipCore.getSharedData().entrySet()) {
if (entry.getValue().getExpireAt() < clock.currentTimeMillis()) {
gossipCore.getSharedData().remove(entry.getKey(), entry.getValue());
}
}
}
void runPerNodeOnce(){
for (Entry<String, ConcurrentHashMap<String, PerNodeDataMessage>> node : gossipCore.getPerNodeData().entrySet()){
void runPerNodeOnce() {
for (Entry<String, ConcurrentHashMap<String, PerNodeDataMessage>> node :
gossipCore.getPerNodeData().entrySet()) {
reapData(node.getValue());
}
}
void reapData(ConcurrentHashMap<String, PerNodeDataMessage> concurrentHashMap){
for (Entry<String, PerNodeDataMessage> entry : concurrentHashMap.entrySet()){
if (entry.getValue().getExpireAt() < clock.currentTimeMillis()){
void reapData(ConcurrentHashMap<String, PerNodeDataMessage> concurrentHashMap) {
for (Entry<String, PerNodeDataMessage> entry : concurrentHashMap.entrySet()) {
if (entry.getValue().getExpireAt() < clock.currentTimeMillis()) {
concurrentHashMap.remove(entry.getKey(), entry.getValue());
}
}
}
public void close(){
public void close() {
scheduledExecutor.shutdown();
try {
scheduledExecutor.awaitTermination(1, TimeUnit.SECONDS);

182
gossip-base/src/main/java/org/apache/gossip/manager/DatacenterRackAwareActiveGossiper.java
View File

@ -31,11 +31,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.gossip.LocalMember;
/**
* Sends gossip traffic at different rates to other racks and data-centers.
* This implementation controls the rate at which gossip traffic is shared.
* There are two constructs Datacenter and Rack. It is assumed that bandwidth and latency is higher
* in the rack than in the the datacenter. We can adjust the rate at which we send messages to each group.
*
* Sends gossip traffic at different rates to other racks and data-centers. This implementation
* controls the rate at which gossip traffic is shared. There are two constructs Datacenter and
* Rack. It is assumed that bandwidth and latency is higher in the rack than in the the datacenter.
* We can adjust the rate at which we send messages to each group.
*/
@Slf4j
public class DatacenterRackAwareActiveGossiper extends AbstractActiveGossiper {
@ -50,124 +49,165 @@ public class DatacenterRackAwareActiveGossiper extends AbstractActiveGossiper {
private ScheduledExecutorService scheduledExecutorService;
private ThreadPoolExecutor threadService;
public DatacenterRackAwareActiveGossiper(GossipManager gossipManager, GossipCore gossipCore,
MetricRegistry registry) {
public DatacenterRackAwareActiveGossiper(
GossipManager gossipManager, GossipCore gossipCore, MetricRegistry registry) {
super(gossipManager, gossipCore, registry);
scheduledExecutorService = Executors.newScheduledThreadPool(2);
workQueue = new ArrayBlockingQueue<Runnable>(1024);
threadService = new ThreadPoolExecutor(1, 30, 1, TimeUnit.SECONDS, workQueue,
new ThreadPoolExecutor.DiscardOldestPolicy());
threadService =
new ThreadPoolExecutor(
1, 30, 1, TimeUnit.SECONDS, workQueue, new ThreadPoolExecutor.DiscardOldestPolicy());
try {
sameRackGossipIntervalMs = Integer.parseInt(gossipManager.getSettings()
.getActiveGossipProperties().get("sameRackGossipIntervalMs"));
} catch (RuntimeException ex) { }
sameRackGossipIntervalMs =
Integer.parseInt(
gossipManager
.getSettings()
.getActiveGossipProperties()
.get("sameRackGossipIntervalMs"));
} catch (RuntimeException ex) {
}
try {
sameDcGossipIntervalMs = Integer.parseInt(gossipManager.getSettings()
.getActiveGossipProperties().get("sameDcGossipIntervalMs"));
} catch (RuntimeException ex) { }
sameDcGossipIntervalMs =
Integer.parseInt(
gossipManager
.getSettings()
.getActiveGossipProperties()
.get("sameDcGossipIntervalMs"));
} catch (RuntimeException ex) {
}
try {
differentDatacenterGossipIntervalMs = Integer.parseInt(gossipManager.getSettings()
.getActiveGossipProperties().get("differentDatacenterGossipIntervalMs"));
} catch (RuntimeException ex) { }
differentDatacenterGossipIntervalMs =
Integer.parseInt(
gossipManager
.getSettings()
.getActiveGossipProperties()
.get("differentDatacenterGossipIntervalMs"));
} catch (RuntimeException ex) {
}
try {
randomDeadMemberSendIntervalMs = Integer.parseInt(gossipManager.getSettings()
.getActiveGossipProperties().get("randomDeadMemberSendIntervalMs"));
} catch (RuntimeException ex) { }
randomDeadMemberSendIntervalMs =
Integer.parseInt(
gossipManager
.getSettings()
.getActiveGossipProperties()
.get("randomDeadMemberSendIntervalMs"));
} catch (RuntimeException ex) {
}
}
@Override
public void init() {
super.init();
//same rack
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> sendToSameRackMember()),
0, sameRackGossipIntervalMs, TimeUnit.MILLISECONDS);
// same rack
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> sendToSameRackMember()),
0,
sameRackGossipIntervalMs,
TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> sendToSameRackMemberPerNode()),
0, sameRackGossipIntervalMs, TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> sendToSameRackMemberPerNode()),
0,
sameRackGossipIntervalMs,
TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> sendToSameRackShared()),
0, sameRackGossipIntervalMs, TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> sendToSameRackShared()),
0,
sameRackGossipIntervalMs,
TimeUnit.MILLISECONDS);
//same dc different rack
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> sameDcDiffernetRackMember()),
0, sameDcGossipIntervalMs, TimeUnit.MILLISECONDS);
// same dc different rack
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> sameDcDiffernetRackMember()),
0,
sameDcGossipIntervalMs,
TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> sameDcDiffernetRackPerNode()),
0, sameDcGossipIntervalMs, TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> sameDcDiffernetRackPerNode()),
0,
sameDcGossipIntervalMs,
TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> sameDcDiffernetRackShared()),
0, sameDcGossipIntervalMs, TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> sameDcDiffernetRackShared()),
0,
sameDcGossipIntervalMs,
TimeUnit.MILLISECONDS);
//different dc
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> differentDcMember()),
0, differentDatacenterGossipIntervalMs, TimeUnit.MILLISECONDS);
// different dc
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> differentDcMember()),
0,
differentDatacenterGossipIntervalMs,
TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> differentDcPerNode()),
0, differentDatacenterGossipIntervalMs, TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> differentDcPerNode()),
0,
differentDatacenterGossipIntervalMs,
TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> differentDcShared()),
0, differentDatacenterGossipIntervalMs, TimeUnit.MILLISECONDS);
//the dead
scheduledExecutorService.scheduleAtFixedRate(() ->
threadService.execute(() -> sendToDeadMember()),
0, randomDeadMemberSendIntervalMs, TimeUnit.MILLISECONDS);
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> differentDcShared()),
0,
differentDatacenterGossipIntervalMs,
TimeUnit.MILLISECONDS);
// the dead
scheduledExecutorService.scheduleAtFixedRate(
() -> threadService.execute(() -> sendToDeadMember()),
0,
randomDeadMemberSendIntervalMs,
TimeUnit.MILLISECONDS);
}
private void sendToDeadMember() {
sendMembershipList(gossipManager.getMyself(), selectPartner(gossipManager.getDeadMembers()));
}
private List<LocalMember> differentDataCenter(){
private List<LocalMember> differentDataCenter() {
String myDc = gossipManager.getMyself().getProperties().get(DATACENTER);
String rack = gossipManager.getMyself().getProperties().get(RACK);
if (myDc == null|| rack == null){
if (myDc == null || rack == null) {
return Collections.emptyList();
}
List<LocalMember> notMyDc = new ArrayList<LocalMember>(10);
for (LocalMember i : gossipManager.getLiveMembers()){
if (!myDc.equals(i.getProperties().get(DATACENTER))){
for (LocalMember i : gossipManager.getLiveMembers()) {
if (!myDc.equals(i.getProperties().get(DATACENTER))) {
notMyDc.add(i);
}
}
return notMyDc;
}
private List<LocalMember> sameDatacenterDifferentRack(){
private List<LocalMember> sameDatacenterDifferentRack() {
String myDc = gossipManager.getMyself().getProperties().get(DATACENTER);
String rack = gossipManager.getMyself().getProperties().get(RACK);
if (myDc == null|| rack == null){
if (myDc == null || rack == null) {
return Collections.emptyList();
}
List<LocalMember> notMyDc = new ArrayList<LocalMember>(10);
for (LocalMember i : gossipManager.getLiveMembers()){
if (myDc.equals(i.getProperties().get(DATACENTER)) && !rack.equals(i.getProperties().get(RACK))){
for (LocalMember i : gossipManager.getLiveMembers()) {
if (myDc.equals(i.getProperties().get(DATACENTER))
&& !rack.equals(i.getProperties().get(RACK))) {
notMyDc.add(i);
}
}
return notMyDc;
}
private List<LocalMember> sameRackNodes(){
private List<LocalMember> sameRackNodes() {
String myDc = gossipManager.getMyself().getProperties().get(DATACENTER);
String rack = gossipManager.getMyself().getProperties().get(RACK);
if (myDc == null|| rack == null){
if (myDc == null || rack == null) {
return Collections.emptyList();
}
List<LocalMember> sameDcAndRack = new ArrayList<LocalMember>(10);
for (LocalMember i : gossipManager.getLiveMembers()){
for (LocalMember i : gossipManager.getLiveMembers()) {
if (myDc.equals(i.getProperties().get(DATACENTER))
&& rack.equals(i.getProperties().get(RACK))){
&& rack.equals(i.getProperties().get(RACK))) {
sameDcAndRack.add(i);
}
}
@ -229,10 +269,8 @@ public class DatacenterRackAwareActiveGossiper extends AbstractActiveGossiper {
}
}
/**
* sends an optimistic shutdown message to several clusters nodes
*/
protected void sendShutdownMessage(){
/** sends an optimistic shutdown message to several clusters nodes */
protected void sendShutdownMessage() {
List<LocalMember> l = gossipManager.getLiveMembers();
int sendTo = l.size() < 3 ? 1 : l.size() / 3;
for (int i = 0; i < sendTo; i++) {

147
gossip-base/src/main/java/org/apache/gossip/manager/GossipCore.java
View File

@ -45,38 +45,41 @@ import org.apache.gossip.udp.Trackable;
public class GossipCore implements GossipCoreConstants {
private final GossipManager gossipManager;
@Getter
private final ConcurrentHashMap<String, ConcurrentHashMap<String, PerNodeDataMessage>> perNodeData;
@Getter
private final ConcurrentHashMap<String, SharedDataMessage> sharedData;
private final ConcurrentHashMap<String, ConcurrentHashMap<String, PerNodeDataMessage>>
perNodeData;
@Getter private final ConcurrentHashMap<String, SharedDataMessage> sharedData;
private final Meter messageSerdeException;
private final Meter transmissionException;
private final Meter transmissionSuccess;
private final DataEventManager eventManager;
private ConcurrentHashMap<String, LatchAndBase> requests;
public GossipCore(GossipManager manager, MetricRegistry metrics){
private final ConcurrentHashMap<String, LatchAndBase> requests;
public GossipCore(GossipManager manager, MetricRegistry metrics) {
this.gossipManager = manager;
requests = new ConcurrentHashMap<>();
perNodeData = new ConcurrentHashMap<>();
sharedData = new ConcurrentHashMap<>();
eventManager = new DataEventManager(metrics);
metrics.register(PER_NODE_DATA_SIZE, (Gauge<Integer>)() -> perNodeData.size());
metrics.register(SHARED_DATA_SIZE, (Gauge<Integer>)() -> sharedData.size());
metrics.register(REQUEST_SIZE, (Gauge<Integer>)() -> requests.size());
metrics.register(PER_NODE_DATA_SIZE, (Gauge<Integer>) perNodeData::size);
metrics.register(SHARED_DATA_SIZE, (Gauge<Integer>) sharedData::size);
metrics.register(REQUEST_SIZE, (Gauge<Integer>) requests::size);
messageSerdeException = metrics.meter(MESSAGE_SERDE_EXCEPTION);
transmissionException = metrics.meter(MESSAGE_TRANSMISSION_EXCEPTION);
transmissionSuccess = metrics.meter(MESSAGE_TRANSMISSION_SUCCESS);
}
@SuppressWarnings({ "unchecked", "rawtypes" })
@SuppressWarnings({"unchecked", "rawtypes"})
public void addSharedData(SharedDataMessage message) {
while (true){
while (true) {
SharedDataMessage previous = sharedData.putIfAbsent(message.getKey(), message);
if (previous == null){
if (previous == null) {
eventManager.notifySharedData(message.getKey(), message.getPayload(), null);
return;
}
if (message.getPayload() instanceof Crdt){
if (message.getPayload() instanceof Crdt) {
SharedDataMessage merged = new SharedDataMessage();
merged.setExpireAt(message.getExpireAt());
merged.setKey(message.getKey());
@ -85,18 +88,19 @@ public class GossipCore implements GossipCoreConstants {
Crdt mergedCrdt = ((Crdt) previous.getPayload()).merge((Crdt) message.getPayload());
merged.setPayload(mergedCrdt);
boolean replaced = sharedData.replace(message.getKey(), previous, merged);
if (replaced){
if(!merged.getPayload().equals(previous.getPayload())) {
eventManager
.notifySharedData(message.getKey(), merged.getPayload(), previous.getPayload());
if (replaced) {
if (!merged.getPayload().equals(previous.getPayload())) {
eventManager.notifySharedData(
message.getKey(), merged.getPayload(), previous.getPayload());
}
return;
}
} else {
if (previous.getTimestamp() < message.getTimestamp()){
if (previous.getTimestamp() < message.getTimestamp()) {
boolean result = sharedData.replace(message.getKey(), previous, message);
if (result){
eventManager.notifySharedData(message.getKey(), message.getPayload(), previous.getPayload());
if (result) {
eventManager.notifySharedData(
message.getKey(), message.getPayload(), previous.getPayload());
return;
}
} else {
@ -106,29 +110,30 @@ public class GossipCore implements GossipCoreConstants {
}
}
public void addPerNodeData(PerNodeDataMessage message){
ConcurrentHashMap<String,PerNodeDataMessage> nodeMap = new ConcurrentHashMap<>();
public void addPerNodeData(PerNodeDataMessage message) {
ConcurrentHashMap<String, PerNodeDataMessage> nodeMap = new ConcurrentHashMap<>();
nodeMap.put(message.getKey(), message);
nodeMap = perNodeData.putIfAbsent(message.getNodeId(), nodeMap);
if (nodeMap != null){
if (nodeMap != null) {
PerNodeDataMessage current = nodeMap.get(message.getKey());
if (current == null){
if (current == null) {
nodeMap.putIfAbsent(message.getKey(), message);
eventManager.notifyPerNodeData(message.getNodeId(), message.getKey(), message.getPayload(), null);
eventManager.notifyPerNodeData(
message.getNodeId(), message.getKey(), message.getPayload(), null);
} else {
if (current.getTimestamp() < message.getTimestamp()){
if (current.getTimestamp() < message.getTimestamp()) {
nodeMap.replace(message.getKey(), current, message);
eventManager.notifyPerNodeData(message.getNodeId(), message.getKey(), message.getPayload(),
current.getPayload());
eventManager.notifyPerNodeData(
message.getNodeId(), message.getKey(), message.getPayload(), current.getPayload());
}
}
} else {
eventManager.notifyPerNodeData(message.getNodeId(), message.getKey(), message.getPayload(), null);
eventManager.notifyPerNodeData(
message.getNodeId(), message.getKey(), message.getPayload(), null);
}
}
public void shutdown(){
}
public void shutdown() {}
public void receive(Base base) {
if (!gossipManager.getMessageHandler().invoke(this, gossipManager, base)) {
@ -137,8 +142,8 @@ public class GossipCore implements GossipCoreConstants {
}
/**
* Sends a blocking message.
* todo: move functionality to TransportManager layer.
* Sends a blocking message. todo: move functionality to TransportManager layer.
*
* @param message
* @param uri
* @throws RuntimeException if data can not be serialized or in transmission error
@ -160,13 +165,13 @@ public class GossipCore implements GossipCoreConstants {
}
}
public Response send(Base message, URI uri){
public Response send(Base message, URI uri) {
log.debug("Sending " + message);
log.debug("Current request queue " + requests);
final Trackable t;
LatchAndBase latchAndBase = null;
if (message instanceof Trackable){
if (message instanceof Trackable) {
t = (Trackable) message;
latchAndBase = new LatchAndBase();
requests.put(t.getUuid() + "/" + t.getUriFrom(), latchAndBase);
@ -174,15 +179,15 @@ public class GossipCore implements GossipCoreConstants {
t = null;
}
sendInternal(message, uri);
if (latchAndBase == null){
if (latchAndBase == null) {
return null;
}
try {
boolean complete = latchAndBase.latch.await(1, TimeUnit.SECONDS);
if (complete){
if (complete) {
return (Response) latchAndBase.base;
} else{
} else {
return null;
}
} catch (InterruptedException e) {
@ -193,8 +198,9 @@ public class GossipCore implements GossipCoreConstants {
}
/**
* Sends a message across the network while blocking. Catches and ignores IOException in transmission. Used
* when the protocol for the message is not to wait for a response
* Sends a message across the network while blocking. Catches and ignores IOException in
* transmission. Used when the protocol for the message is not to wait for a response
*
* @param message the message to send
* @param u the uri to send it to
*/
@ -217,10 +223,9 @@ public class GossipCore implements GossipCoreConstants {
*
* @param senderMember
* @param remoteList
*
*/
public void mergeLists(RemoteMember senderMember, List<Member> remoteList) {
if (log.isDebugEnabled()){
if (log.isDebugEnabled()) {
debugState(senderMember, remoteList);
}
for (LocalMember i : gossipManager.getDeadMembers()) {
@ -228,14 +233,16 @@ public class GossipCore implements GossipCoreConstants {
log.debug(gossipManager.getMyself() + " contacted by dead member " + senderMember.getUri());
i.recordHeartbeat(senderMember.getHeartbeat());
i.setHeartbeat(senderMember.getHeartbeat());
//TODO consider forcing an UP here
// TODO consider forcing an UP here
}
}
for (Member remoteMember : remoteList) {
if (remoteMember.getId().equals(gossipManager.getMyself().getId())) {
continue;
}
LocalMember aNewMember = new LocalMember(remoteMember.getClusterName(),
LocalMember aNewMember =
new LocalMember(
remoteMember.getClusterName(),
remoteMember.getUri(),
remoteMember.getId(),
remoteMember.getHeartbeat(),
@ -245,9 +252,9 @@ public class GossipCore implements GossipCoreConstants {
gossipManager.getSettings().getDistribution());
aNewMember.recordHeartbeat(remoteMember.getHeartbeat());
Object result = gossipManager.getMembers().putIfAbsent(aNewMember, GossipState.UP);
if (result != null){
for (Entry<LocalMember, GossipState> localMember : gossipManager.getMembers().entrySet()){
if (localMember.getKey().getId().equals(remoteMember.getId())){
if (result != null) {
for (Entry<LocalMember, GossipState> localMember : gossipManager.getMembers().entrySet()) {
if (localMember.getKey().getId().equals(remoteMember.getId())) {
localMember.getKey().recordHeartbeat(remoteMember.getHeartbeat());
localMember.getKey().setHeartbeat(remoteMember.getHeartbeat());
localMember.getKey().setProperties(remoteMember.getProperties());
@ -255,28 +262,37 @@ public class GossipCore implements GossipCoreConstants {
}
}
}
if (log.isDebugEnabled()){
if (log.isDebugEnabled()) {
debugState(senderMember, remoteList);
}
}
private void debugState(RemoteMember senderMember,
List<Member> remoteList){
private void debugState(RemoteMember senderMember, List<Member> remoteList) {
log.warn(
"-----------------------\n" +
"Me " + gossipManager.getMyself() + "\n" +
"Sender " + senderMember + "\n" +
"RemoteList " + remoteList + "\n" +
"Live " + gossipManager.getLiveMembers()+ "\n" +
"Dead " + gossipManager.getDeadMembers()+ "\n" +
"=======================");
"-----------------------\n"
+ "Me "
+ gossipManager.getMyself()
+ "\n"
+ "Sender "
+ senderMember
+ "\n"
+ "RemoteList "
+ remoteList
+ "\n"
+ "Live "
+ gossipManager.getLiveMembers()
+ "\n"
+ "Dead "
+ gossipManager.getDeadMembers()
+ "\n"
+ "=======================");
}
@SuppressWarnings("rawtypes")
public Crdt merge(SharedDataMessage message) {
for (;;){
for (; ; ) {
SharedDataMessage previous = sharedData.putIfAbsent(message.getKey(), message);
if (previous == null){
if (previous == null) {
return (Crdt) message.getPayload();
}
SharedDataMessage copy = new SharedDataMessage();
@ -288,35 +304,34 @@ public class GossipCore implements GossipCoreConstants {
Crdt merged = ((Crdt) previous.getPayload()).merge((Crdt) message.getPayload());
copy.setPayload(merged);
boolean replaced = sharedData.replace(message.getKey(), previous, copy);
if (replaced){
if (replaced) {
return merged;
}
}
}
void registerPerNodeDataSubscriber(UpdateNodeDataEventHandler handler){
void registerPerNodeDataSubscriber(UpdateNodeDataEventHandler handler) {
eventManager.registerPerNodeDataSubscriber(handler);
}
void registerSharedDataSubscriber(UpdateSharedDataEventHandler handler){
void registerSharedDataSubscriber(UpdateSharedDataEventHandler handler) {
eventManager.registerSharedDataSubscriber(handler);
}
void unregisterPerNodeDataSubscriber(UpdateNodeDataEventHandler handler){
void unregisterPerNodeDataSubscriber(UpdateNodeDataEventHandler handler) {
eventManager.unregisterPerNodeDataSubscriber(handler);
}
void unregisterSharedDataSubscriber(UpdateSharedDataEventHandler handler){
void unregisterSharedDataSubscriber(UpdateSharedDataEventHandler handler) {
eventManager.unregisterSharedDataSubscriber(handler);
}
class LatchAndBase {
static class LatchAndBase {
private final CountDownLatch latch;
private volatile Base base;
LatchAndBase(){
LatchAndBase() {
latch = new CountDownLatch(1);
}
}
}

174
gossip-base/src/main/java/org/apache/gossip/manager/GossipManager.java
View File

@ -56,15 +56,16 @@ import org.apache.gossip.utils.ReflectionUtils;
@Slf4j
public class GossipManager {
// this mapper is used for ring and user-data persistence only. NOT messages.
public static final ObjectMapper metdataObjectMapper = new ObjectMapper() {
@Serial
private static final long serialVersionUID = 1L;
public static final ObjectMapper metdataObjectMapper =
new ObjectMapper() {
@Serial private static final long serialVersionUID = 1L;
{
enableDefaultTyping();
configure(JsonGenerator.Feature.WRITE_NUMBERS_AS_STRINGS, false);
}};
}
};
private final ConcurrentSkipListMap<LocalMember, GossipState> members;
private final LocalMember me;
@ -83,27 +84,47 @@ public class GossipManager {
private TransportManager transportManager;
private ProtocolManager protocolManager;
public GossipManager(String cluster,
URI uri, String id, Map<String, String> properties, GossipSettings settings,
List<Member> gossipMembers, GossipListener listener, MetricRegistry registry,
public GossipManager(
String cluster,
URI uri,
String id,
Map<String, String> properties,
GossipSettings settings,
List<Member> gossipMembers,
GossipListener listener,
MetricRegistry registry,
MessageHandler messageHandler) {
this.settings = settings;
this.messageHandler = messageHandler;
clock = new SystemClock();
me = new LocalMember(cluster, uri, id, clock.nanoTime(), properties,
settings.getWindowSize(), settings.getMinimumSamples(), settings.getDistribution());
me =
new LocalMember(
cluster,
uri,
id,
clock.nanoTime(),
properties,
settings.getWindowSize(),
settings.getMinimumSamples(),
settings.getDistribution());
gossipCore = new GossipCore(this, registry);
this.lockManager = new LockManager(this, settings.getLockManagerSettings(), registry);
dataReaper = new DataReaper(gossipCore, clock);
members = new ConcurrentSkipListMap<>();
for (Member startupMember : gossipMembers) {
if (!startupMember.equals(me)) {
LocalMember member = new LocalMember(startupMember.getClusterName(),
startupMember.getUri(), startupMember.getId(),
clock.nanoTime(), startupMember.getProperties(), settings.getWindowSize(),
settings.getMinimumSamples(), settings.getDistribution());
//TODO should members start in down state?
LocalMember member =
new LocalMember(
startupMember.getClusterName(),
startupMember.getUri(),
startupMember.getId(),
clock.nanoTime(),
startupMember.getProperties(),
settings.getWindowSize(),
settings.getMinimumSamples(),
settings.getDistribution());
// TODO should members start in down state?
members.put(member, GossipState.DOWN);
}
}
@ -111,28 +132,45 @@ public class GossipManager {
this.scheduledServiced = Executors.newScheduledThreadPool(1);
this.registry = registry;
this.ringState = new RingStatePersister(GossipManager.buildRingStatePath(this), this);
this.userDataState = new UserDataPersister(
this.userDataState =
new UserDataPersister(
gossipCore,
GossipManager.buildPerNodeDataPath(this),
GossipManager.buildSharedDataPath(this));
this.memberStateRefresher = new GossipMemberStateRefresher(members, settings, listener, this::findPerNodeGossipData);
this.memberStateRefresher =
new GossipMemberStateRefresher(members, settings, listener, this::findPerNodeGossipData);
readSavedRingState();
readSavedDataState();
}
public static File buildRingStatePath(GossipManager manager) {
return new File(manager.getSettings().getPathToRingState(), "ringstate." + manager.getMyself().getClusterName() + "."
+ manager.getMyself().getId() + ".json");
return new File(
manager.getSettings().getPathToRingState(),
"ringstate."
+ manager.getMyself().getClusterName()
+ "."
+ manager.getMyself().getId()
+ ".json");
}
public static File buildSharedDataPath(GossipManager manager){
return new File(manager.getSettings().getPathToDataState(), "shareddata."
+ manager.getMyself().getClusterName() + "." + manager.getMyself().getId() + ".json");
public static File buildSharedDataPath(GossipManager manager) {
return new File(
manager.getSettings().getPathToDataState(),
"shareddata."
+ manager.getMyself().getClusterName()
+ "."
+ manager.getMyself().getId()
+ ".json");
}
public static File buildPerNodeDataPath(GossipManager manager) {
return new File(manager.getSettings().getPathToDataState(), "pernodedata."
+ manager.getMyself().getClusterName() + "." + manager.getMyself().getId() + ".json");
return new File(
manager.getSettings().getPathToDataState(),
"pernodedata."
+ manager.getMyself().getClusterName()
+ "."
+ manager.getMyself().getId()
+ ".json");
}
public MessageHandler getMessageHandler() {
@ -152,22 +190,21 @@ public class GossipManager {
*/
public List<LocalMember> getDeadMembers() {
return Collections.unmodifiableList(
members.entrySet()
.stream()
members.entrySet().stream()
.filter(entry -> GossipState.DOWN.equals(entry.getValue()))
.map(Entry::getKey).collect(Collectors.toList()));
.map(Entry::getKey)
.collect(Collectors.toList()));
}
/**
*
* @return a read only list of members found in the UP state
*/
public List<LocalMember> getLiveMembers() {
return Collections.unmodifiableList(
members.entrySet()
.stream()
members.entrySet().stream()
.filter(entry -> GossipState.UP.equals(entry.getValue()))
.map(Entry::getKey).collect(Collectors.toList()));
.map(Entry::getKey)
.collect(Collectors.toList()));
}
public LocalMember getMyself() {
@ -183,17 +220,17 @@ public class GossipManager {
// protocol manager and transport managers are specified in settings.
// construct them here via reflection.
protocolManager = ReflectionUtils.constructWithReflection(
protocolManager =
ReflectionUtils.constructWithReflection(
settings.getProtocolManagerClass(),
new Class<?>[] { GossipSettings.class, String.class, MetricRegistry.class },
new Object[] { settings, me.getId(), this.getRegistry() }
);
new Class<?>[] {GossipSettings.class, String.class, MetricRegistry.class},
new Object[] {settings, me.getId(), this.getRegistry()});
transportManager = ReflectionUtils.constructWithReflection(
transportManager =
ReflectionUtils.constructWithReflection(
settings.getTransportManagerClass(),
new Class<?>[] { GossipManager.class, GossipCore.class},
new Object[] { this, gossipCore }
);
new Class<?>[] {GossipManager.class, GossipCore.class},
new Object[] {this, gossipCore});
// start processing gossip messages.
transportManager.startEndpoint();
@ -213,10 +250,16 @@ public class GossipManager {
private void readSavedRingState() {
if (settings.isPersistRingState()) {
for (LocalMember l : ringState.readFromDisk()) {
LocalMember member = new LocalMember(l.getClusterName(),
l.getUri(), l.getId(),
clock.nanoTime(), l.getProperties(), settings.getWindowSize(),
settings.getMinimumSamples(), settings.getDistribution());
LocalMember member =
new LocalMember(
l.getClusterName(),
l.getUri(),
l.getId(),
clock.nanoTime(),
l.getProperties(),
settings.getWindowSize(),
settings.getMinimumSamples(),
settings.getDistribution());
members.putIfAbsent(member, GossipState.DOWN);
}
}
@ -224,7 +267,8 @@ public class GossipManager {
private void readSavedDataState() {
if (settings.isPersistDataState()) {
for (Entry<String, ConcurrentHashMap<String, PerNodeDataMessage>> l : userDataState.readPerNodeFromDisk().entrySet()) {
for (Entry<String, ConcurrentHashMap<String, PerNodeDataMessage>> l :
userDataState.readPerNodeFromDisk().entrySet()) {
for (Entry<String, PerNodeDataMessage> j : l.getValue().entrySet()) {
gossipCore.addPerNodeData(j.getValue());
}
@ -237,9 +281,7 @@ public class GossipManager {
}
}
/**
* Shutdown the gossip service.
*/
/** Shutdown the gossip service. */
public void shutdown() {
gossipServiceRunning.set(false);
lockManager.shutdown();
@ -256,7 +298,7 @@ public class GossipManager {
scheduledServiced.shutdownNow();
}
public void gossipPerNodeData(PerNodeDataMessage message){
public void gossipPerNodeData(PerNodeDataMessage message) {
Objects.nonNull(message.getKey());
Objects.nonNull(message.getTimestamp());
Objects.nonNull(message.getPayload());
@ -264,7 +306,7 @@ public class GossipManager {
gossipCore.addPerNodeData(message);
}
public void gossipSharedData(SharedDataMessage message){
public void gossipSharedData(SharedDataMessage message) {
Objects.nonNull(message.getKey());
Objects.nonNull(message.getTimestamp());
Objects.nonNull(message.getPayload());
@ -273,12 +315,12 @@ public class GossipManager {
}
@SuppressWarnings("rawtypes")
public Crdt findCrdt(String key){
public Crdt findCrdt(String key) {
SharedDataMessage l = gossipCore.getSharedData().get(key);
if (l == null){
if (l == null) {
return null;
}
if (l.getExpireAt() < clock.currentTimeMillis()){
if (l.getExpireAt() < clock.currentTimeMillis()) {
return null;
} else {
return (Crdt) l.getPayload();
@ -286,24 +328,24 @@ public class GossipManager {
}
@SuppressWarnings("rawtypes")
public Crdt merge(SharedDataMessage message){
public Crdt merge(SharedDataMessage message) {
Objects.nonNull(message.getKey());
Objects.nonNull(message.getTimestamp());
Objects.nonNull(message.getPayload());
message.setNodeId(me.getId());
if (! (message.getPayload() instanceof Crdt)){
if (!(message.getPayload() instanceof Crdt)) {
throw new IllegalArgumentException("Not a subclass of CRDT " + message.getPayload());
}
return gossipCore.merge(message);
}
public PerNodeDataMessage findPerNodeGossipData(String nodeId, String key){
public PerNodeDataMessage findPerNodeGossipData(String nodeId, String key) {
ConcurrentHashMap<String, PerNodeDataMessage> j = gossipCore.getPerNodeData().get(nodeId);
if (j == null){
if (j == null) {
return null;
} else {
PerNodeDataMessage l = j.get(key);
if (l == null){
if (l == null) {
return null;
}
if (l.getExpireAt() != null && l.getExpireAt() < clock.currentTimeMillis()) {
@ -313,12 +355,12 @@ public class GossipManager {
}
}
public SharedDataMessage findSharedGossipData(String key){
public SharedDataMessage findSharedGossipData(String key) {
SharedDataMessage l = gossipCore.getSharedData().get(key);
if (l == null){
if (l == null) {
return null;
}
if (l.getExpireAt() < clock.currentTimeMillis()){
if (l.getExpireAt() < clock.currentTimeMillis()) {
return null;
} else {
return l;
@ -359,19 +401,19 @@ public class GossipManager {
return transportManager;
}
public void registerPerNodeDataSubscriber(UpdateNodeDataEventHandler handler){
public void registerPerNodeDataSubscriber(UpdateNodeDataEventHandler handler) {
gossipCore.registerPerNodeDataSubscriber(handler);
}
public void registerSharedDataSubscriber(UpdateSharedDataEventHandler handler){
public void registerSharedDataSubscriber(UpdateSharedDataEventHandler handler) {
gossipCore.registerSharedDataSubscriber(handler);
}
public void unregisterPerNodeDataSubscriber(UpdateNodeDataEventHandler handler){
public void unregisterPerNodeDataSubscriber(UpdateNodeDataEventHandler handler) {
gossipCore.unregisterPerNodeDataSubscriber(handler);
}
public void unregisterSharedDataSubscriber(UpdateSharedDataEventHandler handler){
public void unregisterSharedDataSubscriber(UpdateSharedDataEventHandler handler) {
gossipCore.unregisterSharedDataSubscriber(handler);
}
@ -381,6 +423,7 @@ public class GossipManager {
/**
* Get the lock manager specified with this GossipManager.
*
* @return lock manager object.
*/
public LockManager getLockManager() {
@ -389,10 +432,11 @@ public class GossipManager {
/**
* Try to acquire a lock on given shared data key.
*
* @param key key of tha share data object.
* @throws VoteFailedException if the locking is failed.
*/
public void acquireSharedDataLock(String key) throws VoteFailedException{
public void acquireSharedDataLock(String key) throws VoteFailedException {
lockManager.acquireSharedDataLock(key);
}
}

14
gossip-base/src/main/java/org/apache/gossip/manager/GossipManagerBuilder.java
View File

@ -18,19 +18,17 @@
package org.apache.gossip.manager;
import com.codahale.metrics.MetricRegistry;
import org.apache.gossip.GossipSettings;
import org.apache.gossip.Member;
import org.apache.gossip.StartupSettings;
import org.apache.gossip.event.GossipListener;
import org.apache.gossip.event.GossipState;
import org.apache.gossip.manager.handlers.MessageHandler;
import org.apache.gossip.manager.handlers.MessageHandlerFactory;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.gossip.GossipSettings;
import org.apache.gossip.Member;
import org.apache.gossip.StartupSettings;
import org.apache.gossip.event.GossipListener;
import org.apache.gossip.manager.handlers.MessageHandler;
import org.apache.gossip.manager.handlers.MessageHandlerFactory;
public class GossipManagerBuilder {

29
gossip-base/src/main/java/org/apache/gossip/manager/GossipMemberStateRefresher.java
View File

@ -43,7 +43,9 @@ public class GossipMemberStateRefresher {
private final ScheduledExecutorService scheduledExecutor;
private final BlockingQueue<Runnable> workQueue;
public GossipMemberStateRefresher(Map<LocalMember, GossipState> members, GossipSettings settings,
public GossipMemberStateRefresher(
Map<LocalMember, GossipState> members,
GossipSettings settings,
GossipListener listener,
BiFunction<String, String, PerNodeDataMessage> findPerNodeGossipData) {
this.members = members;
@ -52,8 +54,9 @@ public class GossipMemberStateRefresher {
this.findPerNodeGossipData = findPerNodeGossipData;
clock = new SystemClock();
workQueue = new ArrayBlockingQueue<>(1024);
listenerExecutor = new ThreadPoolExecutor(1, 20, 1, TimeUnit.SECONDS, workQueue,
new ThreadPoolExecutor.DiscardOldestPolicy());
listenerExecutor =
new ThreadPoolExecutor(
1, 20, 1, TimeUnit.SECONDS, workQueue, new ThreadPoolExecutor.DiscardOldestPolicy());
scheduledExecutor = Executors.newScheduledThreadPool(1);
}
@ -72,8 +75,7 @@ public class GossipMemberStateRefresher {
public void runOnce() {
for (Entry<LocalMember, GossipState> entry : members.entrySet()) {
boolean userDown = processOptimisticShutdown(entry);
if (userDown)
continue;
if (userDown) continue;
Double phiMeasure = entry.getKey().detect(clock.nanoTime());
GossipState requiredState;
@ -87,17 +89,15 @@ public class GossipMemberStateRefresher {
if (entry.getValue() != requiredState) {
members.put(entry.getKey(), requiredState);
/* Call listeners asynchronously */
for (GossipListener listener: listeners)
for (GossipListener listener : listeners)
listenerExecutor.execute(() -> listener.gossipEvent(entry.getKey(), requiredState));
}
}
}
public GossipState calcRequiredState(Double phiMeasure) {
if (phiMeasure > settings.getConvictThreshold())
return GossipState.DOWN;
else
return GossipState.UP;
if (phiMeasure > settings.getConvictThreshold()) return GossipState.DOWN;
else return GossipState.UP;
}
public GossipState calcRequiredStateCleanupInterval(LocalMember member, GossipState state) {
@ -111,14 +111,15 @@ public class GossipMemberStateRefresher {
}
/**
* If we have a special key the per-node data that means that the node has sent us
* a pre-emptive shutdown message. We process this so node is seen down sooner
* If we have a special key the per-node data that means that the node has sent us a pre-emptive
* shutdown message. We process this so node is seen down sooner
*
* @param l member to consider
* @return true if node forced down
*/
public boolean processOptimisticShutdown(Entry<LocalMember, GossipState> l) {
PerNodeDataMessage m = findPerNodeGossipData.apply(l.getKey().getId(), ShutdownMessage.PER_NODE_KEY);
PerNodeDataMessage m =
findPerNodeGossipData.apply(l.getKey().getId(), ShutdownMessage.PER_NODE_KEY);
if (m == null) {
return false;
}
@ -126,7 +127,7 @@ public class GossipMemberStateRefresher {
if (s.getShutdownAtNanos() > l.getKey().getHeartbeat()) {
members.put(l.getKey(), GossipState.DOWN);
if (l.getValue() == GossipState.UP) {
for (GossipListener listener: listeners)
for (GossipListener listener : listeners)
listenerExecutor.execute(() -> listener.gossipEvent(l.getKey(), GossipState.DOWN));
}
return true;

6
gossip-base/src/main/java/org/apache/gossip/manager/RingStatePersister.java
View File

@ -36,7 +36,7 @@ public class RingStatePersister implements Runnable {
private final ObjectMapper objectMapper;
private final GossipManager manager;
public RingStatePersister(File path, GossipManager manager){
public RingStatePersister(File path, GossipManager manager) {
this.path = path;
this.objectMapper = GossipManager.metdataObjectMapper;
this.manager = manager;
@ -49,7 +49,7 @@ public class RingStatePersister implements Runnable {
void writeToDisk() {
NavigableSet<LocalMember> i = manager.getMembers().keySet();
try (FileOutputStream fos = new FileOutputStream(path)){
try (FileOutputStream fos = new FileOutputStream(path)) {
objectMapper.writeValue(fos, i);
} catch (IOException e) {
log.error("Error!", e);
@ -61,7 +61,7 @@ public class RingStatePersister implements Runnable {
if (!path.exists()) {
return new ArrayList<>();
}
try (FileInputStream fos = new FileInputStream(path)){
try (FileInputStream fos = new FileInputStream(path)) {
return objectMapper.readValue(fos, ArrayList.class);
} catch (IOException e) {
log.error("Error", e);

4
gossip-base/src/main/java/org/apache/gossip/manager/SimpleActiveGossiper.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.gossip.manager;
import com.codahale.metrics.MetricRegistry;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
@ -24,12 +25,9 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.gossip.LocalMember;
import com.codahale.metrics.MetricRegistry;
/** Base implementation gossips randomly to live nodes periodically gossips to dead ones */
@Slf4j
public class SimpleActiveGossiper extends AbstractActiveGossiper {

8
gossip-base/src/main/java/org/apache/gossip/manager/UserDataPersister.java
View File

@ -17,14 +17,12 @@
*/
package org.apache.gossip.manager;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.java.Log;
import lombok.extern.slf4j.Slf4j;
import org.apache.gossip.model.PerNodeDataMessage;
import org.apache.gossip.model.SharedDataMessage;
@ -48,14 +46,14 @@ public class UserDataPersister implements Runnable {
@SuppressWarnings("unchecked")
ConcurrentHashMap<String, ConcurrentHashMap<String, PerNodeDataMessage>> readPerNodeFromDisk() {
if (!perNodePath.exists()) {
return new ConcurrentHashMap<String, ConcurrentHashMap<String, PerNodeDataMessage>>();
return new ConcurrentHashMap<>();
}
try (FileInputStream fos = new FileInputStream(perNodePath)) {
return objectMapper.readValue(fos, ConcurrentHashMap.class);
} catch (IOException e) {
log.error("Error!", e);
}
return new ConcurrentHashMap<String, ConcurrentHashMap<String, PerNodeDataMessage>>();
return new ConcurrentHashMap<>();
}
void writePerNodeToDisk() {

3
gossip-base/src/main/java/org/apache/gossip/manager/handlers/ActiveGossipMessageHandler.java
View File

@ -53,7 +53,8 @@ public class ActiveGossipMessageHandler implements MessageHandler {
log.debug("Gossip message with faulty URI", e);
continue;
}
RemoteMember member = new RemoteMember(
RemoteMember member =
new RemoteMember(
activeGossipMessage.getMembers().get(i).getCluster(),
u,
activeGossipMessage.getMembers().get(i).getId(),

11
gossip-base/src/main/java/org/apache/gossip/model/Base.java
View File

@ -26,10 +26,7 @@ import org.apache.gossip.udp.UdpNotAMemberFault;
import org.apache.gossip.udp.UdpPerNodeDataBulkMessage;
import org.apache.gossip.udp.UdpSharedDataBulkMessage;
@JsonTypeInfo(
use = JsonTypeInfo.Id.CLASS,
include = JsonTypeInfo.As.PROPERTY,
property = "type")
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "type")
@JsonSubTypes({
@Type(value = ActiveGossipMessage.class, name = "ActiveGossipMessage"),
@Type(value = Fault.class, name = "Fault"),
@ -41,7 +38,5 @@ import org.apache.gossip.udp.UdpSharedDataBulkMessage;
@Type(value = UdpPerNodeDataBulkMessage.class, name = "UdpPerNodeDataMessage"),
@Type(value = SharedDataMessage.class, name = "SharedDataMessage"),
@Type(value = UdpSharedDataBulkMessage.class, name = "UdpSharedDataMessage")
})
public class Base {
}
})
public class Base {}

26
gossip-base/src/main/java/org/apache/gossip/model/SharedDataMessage.java
View File

@ -31,30 +31,39 @@ public class SharedDataMessage extends Base {
public String getNodeId() {
return nodeId;
}
public void setNodeId(String nodeId) {
this.nodeId = nodeId;
}
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public Object getPayload() {
return payload;
}
public void setPayload(Object payload) {
this.payload = payload;
}
public Long getTimestamp() {
return timestamp;
}
public void setTimestamp(Long timestamp) {
this.timestamp = timestamp;
}
public Long getExpireAt() {
return expireAt;
}
public void setExpireAt(Long expireAt) {
this.expireAt = expireAt;
}
@ -69,9 +78,18 @@ public class SharedDataMessage extends Base {
@Override
public String toString() {
return "SharedGossipDataMessage [nodeId=" + nodeId + ", key=" + key + ", payload=" + payload
+ ", timestamp=" + timestamp + ", expireAt=" + expireAt
+ ", replicable=" + replicable + "]";
return "SharedGossipDataMessage [nodeId="
+ nodeId
+ ", key="
+ key
+ ", payload="
+ payload
+ ", timestamp="
+ timestamp
+ ", expireAt="
+ expireAt
+ ", replicable="
+ replicable
+ "]";
}
}

5
gossip-base/src/main/java/org/apache/gossip/protocol/ProtocolManager.java
View File

@ -23,7 +23,9 @@ import org.apache.gossip.model.Base;
/** interface for managing message marshaling. */
public interface ProtocolManager {
/** serialize a message
/**
* serialize a message
*
* @param message
* @return serialized message.
* @throws IOException
@ -32,6 +34,7 @@ public interface ProtocolManager {
/**
* Reads the next message from a byte source.
*
* @param buf
* @return a gossip message.
* @throws IOException

5
gossip-base/src/main/java/org/apache/gossip/replication/BlackListReplicable.java
View File

@ -17,11 +17,10 @@
*/
package org.apache.gossip.replication;
import org.apache.gossip.LocalMember;
import org.apache.gossip.model.Base;
import java.util.ArrayList;
import java.util.List;
import org.apache.gossip.LocalMember;
import org.apache.gossip.model.Base;
/**
* Replicable implementation which does not replicate data to given set of nodes.

15
gossip-base/src/main/java/org/apache/gossip/transport/AbstractTransportManager.java
View File

@ -27,9 +27,7 @@ import org.apache.gossip.manager.GossipCore;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.utils.ReflectionUtils;
/**
* Manage the protcol threads (active and passive gossipers).
*/
/** Manage the protcol threads (active and passive gossipers). */
@Slf4j
public abstract class AbstractTransportManager implements TransportManager {
@ -42,14 +40,11 @@ public abstract class AbstractTransportManager implements TransportManager {
this.gossipManager = gossipManager;
this.gossipCore = gossipCore;
gossipThreadExecutor = Executors.newCachedThreadPool();
activeGossipThread = ReflectionUtils.constructWithReflection(
activeGossipThread =
ReflectionUtils.constructWithReflection(
gossipManager.getSettings().getActiveGossipClass(),
new Class<?>[]{
GossipManager.class, GossipCore.class, MetricRegistry.class
},
new Object[]{
gossipManager, gossipCore, gossipManager.getRegistry()
});
new Class<?>[] {GossipManager.class, GossipCore.class, MetricRegistry.class},
new Object[] {gossipManager, gossipCore, gossipManager.getRegistry()});
}
// shut down threads etc.

1
gossip-base/src/main/java/org/apache/gossip/udp/UdpNotAMemberFault.java
View File

@ -23,6 +23,7 @@ public class UdpNotAMemberFault extends NotAMemberFault implements Trackable {
private String uriFrom;
private String uuid;
public UdpNotAMemberFault() {}
public String getUriFrom() {

1
gossip-base/src/test/java/org/apache/gossip/AbstractIntegrationBase.java
View File

@ -23,7 +23,6 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
import org.junit.After;

6
gossip-base/src/test/java/org/apache/gossip/MemberTest.java
View File

@ -27,7 +27,8 @@ public class MemberTest {
@Test
public void testHashCodeFromGossip40() throws URISyntaxException {
Assertions.assertNotEquals(new LocalMember(
Assertions.assertNotEquals(
new LocalMember(
"mycluster",
new URI("udp://4.4.4.4:1000"),
"myid",
@ -36,7 +37,8 @@ public class MemberTest {
10,
5,
"exponential")
.hashCode(), new LocalMember(
.hashCode(),
new LocalMember(
"mycluster",
new URI("udp://4.4.4.5:1005"),
"yourid",

18
gossip-base/src/test/java/org/apache/gossip/accrual/FailureDetectorTest.java
View File

@ -54,11 +54,13 @@ public class FailureDetectorTest {
}
Integer lastRecorded = values.get(values.size() - 2);
//after "step" delay we need to be considered UP
// after "step" delay we need to be considered UP
Assertions.assertTrue(fd.computePhiMeasure(values.get(values.size() - 1)) < failureThreshold);
//if we check phi-measure after mean delay we get value for 0.5 probability(normal distribution)
Assertions.assertEquals(fd.computePhiMeasure(lastRecorded + Math.round(deltaSum / deltaCount)),
// if we check phi-measure after mean delay we get value for 0.5 probability(normal
// distribution)
Assertions.assertEquals(
fd.computePhiMeasure(lastRecorded + Math.round(deltaSum / deltaCount)),
-Math.log10(0.5),
0.1);
}
@ -67,7 +69,9 @@ public class FailureDetectorTest {
public void checkMinimumSamples() {
int minimumSamples = 5;
FailureDetector fd = new FailureDetector(minimumSamples, 1000, "normal");
for (int i = 0; i < minimumSamples + 1; i++) { // +1 because we don't place first heartbeat into structure
for (int i = 0;
i < minimumSamples + 1;
i++) { // +1 because we don't place first heartbeat into structure
Assertions.assertNull(fd.computePhiMeasure(100));
fd.recordHeartbeat(i);
}
@ -77,7 +81,8 @@ public class FailureDetectorTest {
@Test
public void checkMonotonicDead() {
final FailureDetector fd = new FailureDetector(5, 1000, "normal");
TriConsumer<Integer, Integer, Integer> checkAlive = (begin, end, step) -> {
TriConsumer<Integer, Integer, Integer> checkAlive =
(begin, end, step) -> {
List<Integer> times = generateTimeList(begin, end, step);
for (Integer time : times) {
Double current = fd.computePhiMeasure(time);
@ -88,7 +93,8 @@ public class FailureDetectorTest {
}
};
TriConsumer<Integer, Integer, Integer> checkDeadMonotonic = (begin, end, step) -> {
TriConsumer<Integer, Integer, Integer> checkDeadMonotonic =
(begin, end, step) -> {
List<Integer> times = generateTimeList(begin, end, step);
Double prev = null;
for (Integer time : times) {

9
gossip-base/src/test/java/org/apache/gossip/crdt/AddRemoveStringSetTest.java
View File

@ -17,15 +17,14 @@
*/
package org.apache.gossip.crdt;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
/*
Abstract test suit to test CrdtSets with Add and Remove operations.

1
gossip-base/src/test/java/org/apache/gossip/crdt/GrowOnlySetTest.java
View File

@ -19,7 +19,6 @@ package org.apache.gossip.crdt;
import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
import org.junit.Assert;
import org.junit.Test;

9
gossip-base/src/test/java/org/apache/gossip/crdt/LwwSetTest.java
View File

@ -17,15 +17,14 @@
*/
package org.apache.gossip.crdt;
import org.apache.gossip.manager.Clock;
import org.apache.gossip.manager.SystemClock;
import org.junit.Assert;
import org.junit.Test;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.gossip.manager.Clock;
import org.apache.gossip.manager.SystemClock;
import org.junit.Assert;
import org.junit.Test;
public class LwwSetTest extends AddRemoveStringSetTest<LwwSet<String>> {
private static Clock clock = new SystemClock();

8
gossip-base/src/test/java/org/apache/gossip/crdt/MaxChangeSetTest.java
View File

@ -25,16 +25,16 @@ import org.junit.Assert;
import org.junit.Test;
public class MaxChangeSetTest extends AddRemoveStringSetTest<MaxChangeSet<String>> {
MaxChangeSet<String> construct(Set<String> set){
MaxChangeSet<String> construct(Set<String> set) {
return new MaxChangeSet<>(set);
}
MaxChangeSet<String> construct(){
MaxChangeSet<String> construct() {
return new MaxChangeSet<>();
}
@Test
public void valueTest(){
public void valueTest() {
Map<Character, Integer> struct = new HashMap<>();
struct.put('a', 0);
struct.put('b', 1);
@ -47,7 +47,7 @@ public class MaxChangeSetTest extends AddRemoveStringSetTest<MaxChangeSet<String
}
@Test
public void mergeTest(){
public void mergeTest() {
MaxChangeSet<Integer> set1 = new MaxChangeSet<Integer>().add(1); // Set with one operation on 1
MaxChangeSet<Integer> set2 = new MaxChangeSet<Integer>().add(1).remove(1); // two operations
Assert.assertEquals(set1.merge(set2), new MaxChangeSet<Integer>()); // empty set wins

32
gossip-base/src/test/java/org/apache/gossip/crdt/OrSetTest.java
View File

@ -25,22 +25,22 @@ import org.junit.Assert;
import org.junit.Test;
public class OrSetTest extends AddRemoveStringSetTest<OrSet<String>> {
OrSet<String> construct(){
OrSet<String> construct() {
return new OrSet<>();
}
OrSet<String> construct(Set<String> set){
OrSet<String> construct(Set<String> set) {
return new OrSet<>(set);
}
@Test
public void atest(){
public void atest() {
OrSet<Integer> i = new OrSet<>(new OrSet.Builder<Integer>().add(4).add(5).add(6).remove(5));
Assert.assertArrayEquals(Arrays.asList(4, 6).toArray(), i.value().toArray());
}
@Test
public void mergeTest(){
public void mergeTest() {
OrSet<Integer> i = new OrSet<>(new OrSet.Builder<Integer>().add(4).add(5).add(6).remove(5));
Assert.assertArrayEquals(Arrays.asList(4, 6).toArray(), i.value().toArray());
OrSet<Integer> j = new OrSet<>(new OrSet.Builder<Integer>().add(9).add(4).add(5).remove(6));
@ -49,11 +49,11 @@ public class OrSetTest extends AddRemoveStringSetTest<OrSet<String>> {
}
@Test
public void mergeTest2(){
public void mergeTest2() {
OrSet<Integer> i = new OrSet<>(new OrSet.Builder<Integer>().add(5).add(4).remove(4).add(6));
Assert.assertEquals(new OrSet<>(5, 6), i);
SortedSet<Integer> tree = new TreeSet<>();
for (Integer in : i.value()){
for (Integer in : i.value()) {
tree.add(in);
}
TreeSet<Integer> compare = new TreeSet<>();
@ -63,32 +63,32 @@ public class OrSetTest extends AddRemoveStringSetTest<OrSet<String>> {
}
@Test
public void mergeTest4(){
Assert.assertArrayEquals(new Integer[]{},
new OrSet<>(new OrSet.Builder<Integer>().add(1).remove(1)).toArray());
public void mergeTest4() {
Assert.assertArrayEquals(
new Integer[] {}, new OrSet<>(new OrSet.Builder<Integer>().add(1).remove(1)).toArray());
}
@Test
public void mergeTest3(){
public void mergeTest3() {
OrSet<Integer> i = new OrSet<>(1);
OrSet<Integer> j = new OrSet<>(2);
OrSet<Integer> k = new OrSet<>(i.merge(j), new OrSet.Builder<Integer>().remove(1));
Assert.assertArrayEquals(new Integer[]{2}, i.merge(j).merge(k).toArray());
Assert.assertArrayEquals(new Integer[]{2}, j.merge(i).merge(k).toArray());
Assert.assertArrayEquals(new Integer[]{2}, k.merge(i).merge(j).toArray());
Assert.assertArrayEquals(new Integer[]{2}, k.merge(j).merge(i).toArray());
Assert.assertArrayEquals(new Integer[] {2}, i.merge(j).merge(k).toArray());
Assert.assertArrayEquals(new Integer[] {2}, j.merge(i).merge(k).toArray());
Assert.assertArrayEquals(new Integer[] {2}, k.merge(i).merge(j).toArray());
Assert.assertArrayEquals(new Integer[] {2}, k.merge(j).merge(i).toArray());
Assert.assertEquals(j, i.merge(j.merge(k)));
}
@Test
public void mergeTest9(){
public void mergeTest9() {
OrSet<Integer> i = new OrSet<>(19);
OrSet<Integer> j = i.merge(i);
Assert.assertEquals(i.value(), j.value());
}
@Test
public void mergeTestSame(){
public void mergeTestSame() {
OrSet<Integer> i = new OrSet<>(19);
OrSet<Integer> j = new OrSet<>(19);
OrSet<Integer> k = i.merge(j);

1
gossip-base/src/test/java/org/apache/gossip/crdt/PNCounterTest.java
View File

@ -22,7 +22,6 @@ import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.List;
import org.apache.gossip.LocalMember;
import org.apache.gossip.manager.GossipManager;
import org.junit.Assert;

7
gossip-base/src/test/java/org/apache/gossip/crdt/TwoPhaseSetTest.java
View File

@ -17,13 +17,12 @@
*/
package org.apache.gossip.crdt;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.HashSet;
import java.util.Set;
import java.util.function.BiConsumer;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class TwoPhaseSetTest {

5
gossip-base/src/test/java/org/apache/gossip/event/data/DataEventManagerTest.java
View File

@ -18,13 +18,12 @@
package org.apache.gossip.event.data;
import com.codahale.metrics.MetricRegistry;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
public class DataEventManagerTest {
private static Semaphore semaphore;

5
gossip-base/src/test/java/org/apache/gossip/lock/vote/MajorityVoteTest.java
View File

@ -17,12 +17,11 @@
*/
package org.apache.gossip.lock.vote;
import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
public class MajorityVoteTest {

4
gossip-base/src/test/java/org/apache/gossip/manager/DataReaperTest.java
View File

@ -18,16 +18,14 @@
package org.apache.gossip.manager;
import com.codahale.metrics.MetricRegistry;
import io.teknek.tunit.TUnit;
import java.net.URI;
import org.apache.gossip.GossipSettings;
import org.apache.gossip.model.PerNodeDataMessage;
import org.apache.gossip.model.SharedDataMessage;
import org.junit.Assert;
import org.junit.Test;
import io.teknek.tunit.TUnit;
public class DataReaperTest {
private final MetricRegistry registry = new MetricRegistry();

1
gossip-base/src/test/java/org/apache/gossip/manager/UserDataPersistenceTest.java
View File

@ -22,7 +22,6 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.gossip.GossipSettings;
import org.apache.gossip.model.PerNodeDataMessage;
import org.apache.gossip.model.SharedDataMessage;

10
gossip-base/src/test/java/org/apache/gossip/manager/handlers/MessageHandlerTest.java
View File

@ -28,7 +28,8 @@ import org.junit.Test;
public class MessageHandlerTest {
@Test
public void testSimpleHandler() {
MessageHandler mi = new TypedMessageHandlerWrapper(FakeMessage.class, new DummyMessageHandler());
MessageHandler mi =
new TypedMessageHandlerWrapper(FakeMessage.class, new DummyMessageHandler());
Assert.assertTrue(mi.invoke(null, null, new FakeMessage()));
Assert.assertFalse(mi.invoke(null, null, new ActiveGossipMessage()));
}
@ -72,7 +73,8 @@ public class MessageHandlerTest {
@Test
public void testMessageHandlerCombiner() {
// Empty combiner - false result
MessageHandler mi = MessageHandlerFactory.concurrentHandler((gossipCore, gossipManager, base) -> false);
MessageHandler mi =
MessageHandlerFactory.concurrentHandler((gossipCore, gossipManager, base) -> false);
Assert.assertFalse(mi.invoke(null, null, new Base()));
DummyMessageHandler h = new DummyMessageHandler();
@ -86,7 +88,9 @@ public class MessageHandlerTest {
Assert.assertEquals(2, h.counter);
// Increase size in runtime. Should be 3 calls: 2+3 = 5
mi = MessageHandlerFactory.concurrentHandler(mi, new TypedMessageHandlerWrapper(FakeMessage.class, h));
mi =
MessageHandlerFactory.concurrentHandler(
mi, new TypedMessageHandlerWrapper(FakeMessage.class, h));
Assert.assertTrue(mi.invoke(null, null, new FakeMessage()));
Assert.assertEquals(5, h.counter);
}

26
gossip-base/src/test/java/org/apache/gossip/protocol/UnitTestProtocolManager.java
View File

@ -30,14 +30,16 @@ import org.apache.gossip.model.Base;
// doesn't serialize anything besides longs. Uses a static lookup table to read and write objects.
public class UnitTestProtocolManager implements ProtocolManager {
// so it can be shared across gossipers. this works as long as each object has a different memory address.
// so it can be shared across gossipers. this works as long as each object has a different memory
// address.
private static final Map<Long, Base> lookup = new ConcurrentHashMap<>();
private final Meter meter;
public UnitTestProtocolManager(GossipSettings settings, String id, MetricRegistry registry) {
meter = settings.isSignMessages() ?
registry.meter(PassiveGossipConstants.SIGNED_MESSAGE) :
registry.meter(PassiveGossipConstants.UNSIGNED_MESSAGE);
meter =
settings.isSignMessages()
? registry.meter(PassiveGossipConstants.SIGNED_MESSAGE)
: registry.meter(PassiveGossipConstants.UNSIGNED_MESSAGE);
}
private static byte[] longToBytes(long val) {
@ -54,14 +56,14 @@ public class UnitTestProtocolManager implements ProtocolManager {
}
static long bytesToLong(byte[] b) {
return ((b[7] & 0xFFL)) +
((b[6] & 0xFFL) << 8) +
((b[5] & 0xFFL) << 16) +
((b[4] & 0xFFL) << 24) +
((b[3] & 0xFFL) << 32) +
((b[2] & 0xFFL) << 40) +
((b[1] & 0xFFL) << 48) +
(((long) b[0]) << 56);
return ((b[7] & 0xFFL))
+ ((b[6] & 0xFFL) << 8)
+ ((b[5] & 0xFFL) << 16)
+ ((b[4] & 0xFFL) << 24)
+ ((b[3] & 0xFFL) << 32)
+ ((b[2] & 0xFFL) << 40)
+ ((b[1] & 0xFFL) << 48)
+ (((long) b[0]) << 56);
}
@Override

160
gossip-base/src/test/java/org/apache/gossip/replication/DataReplicationTest.java
View File

@ -31,8 +31,8 @@ import org.junit.Test;
public class DataReplicationTest {
private static SharedDataMessage getSharedNodeData(String key, String value,
Replicable<SharedDataMessage> replicable) {
private static SharedDataMessage getSharedNodeData(
String key, String value, Replicable<SharedDataMessage> replicable) {
SharedDataMessage g = new SharedDataMessage();
g.setExpireAt(Long.MAX_VALUE);
g.setKey(key);
@ -42,11 +42,11 @@ public class DataReplicationTest {
return g;
}
private static LocalMember getLocalMember(URI uri, String id){
private static LocalMember getLocalMember(URI uri, String id) {
return new LocalMember("cluster1", uri, id, 0, null, 1, 0, "");
}
private static LocalMember getLocalMemberDc(URI uri, String id, String dataCenter, String rack){
private static LocalMember getLocalMemberDc(URI uri, String id, String dataCenter, String rack) {
Map<String, String> props = new HashMap<>();
props.put(DatacenterRackAwareActiveGossiper.DATACENTER, dataCenter);
props.put(DatacenterRackAwareActiveGossiper.RACK, rack);
@ -55,99 +55,103 @@ public class DataReplicationTest {
@Test
public void dataReplicateAllTest() throws URISyntaxException {
SharedDataMessage message = getSharedNodeData("public","public", new AllReplicable<>());
LocalMember me = getLocalMember(new URI("udp://127.0.0.1:8001"),"1");
LocalMember member = getLocalMember(new URI("udp://127.0.0.1:8002"),"2");
SharedDataMessage message = getSharedNodeData("public", "public", new AllReplicable<>());
LocalMember me = getLocalMember(new URI("udp://127.0.0.1:8001"), "1");
LocalMember member = getLocalMember(new URI("udp://127.0.0.1:8002"), "2");
Assert.assertEquals(true, message.getReplicable().shouldReplicate(me, member, message));
}
@Test
public void dataReplicateNoneTest() throws URISyntaxException {
SharedDataMessage message = getSharedNodeData("private","private", new NotReplicable<>());
LocalMember me = getLocalMember(new URI("udp://127.0.0.1:8001"),"1");
LocalMember member = getLocalMember(new URI("udp://127.0.0.1:8002"),"2");
SharedDataMessage message = getSharedNodeData("private", "private", new NotReplicable<>());
LocalMember me = getLocalMember(new URI("udp://127.0.0.1:8001"), "1");
LocalMember member = getLocalMember(new URI("udp://127.0.0.1:8002"), "2");
Assert.assertEquals(false, message.getReplicable().shouldReplicate(me, member, message));
}
@Test
public void dataReplicateWhiteListTest() throws URISyntaxException {
List<LocalMember> memberList = new ArrayList<>();
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8001"),"1"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8002"),"2"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8003"),"3"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8001"), "1"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8002"), "2"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8003"), "3"));
// add node 1 and 2 to the white list
List<LocalMember> whiteList = new ArrayList<>();
whiteList.add(memberList.get(0));
whiteList.add(memberList.get(1));
SharedDataMessage message = getSharedNodeData("whiteList", "Only allow some nodes",
new WhiteListReplicable<>(whiteList));
LocalMember me = getLocalMember(new URI("udp://127.0.0.1:8004"),"4");
SharedDataMessage message =
getSharedNodeData(
"whiteList", "Only allow some nodes", new WhiteListReplicable<>(whiteList));
LocalMember me = getLocalMember(new URI("udp://127.0.0.1:8004"), "4");
// data should replicate to node 1 and 2 but not 3
Assert.assertEquals(true,
message.getReplicable().shouldReplicate(me, memberList.get(0), message));
Assert.assertEquals(true,
message.getReplicable().shouldReplicate(me, memberList.get(1), message));
Assert.assertEquals(false,
message.getReplicable().shouldReplicate(me, memberList.get(2), message));
Assert.assertEquals(
true, message.getReplicable().shouldReplicate(me, memberList.get(0), message));
Assert.assertEquals(
true, message.getReplicable().shouldReplicate(me, memberList.get(1), message));
Assert.assertEquals(
false, message.getReplicable().shouldReplicate(me, memberList.get(2), message));
}
@Test
public void dataReplicateWhiteListNullTest() throws URISyntaxException {
List<LocalMember> memberList = new ArrayList<>();
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8001"),"1"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8002"),"2"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8001"), "1"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8002"), "2"));
SharedDataMessage message = getSharedNodeData("whiteList", "Only allow some nodes",
new WhiteListReplicable<>(null));
SharedDataMessage message =
getSharedNodeData("whiteList", "Only allow some nodes", new WhiteListReplicable<>(null));
// data should not replicate if no whitelist specified
Assert.assertEquals(false,
Assert.assertEquals(
false,
message.getReplicable().shouldReplicate(memberList.get(0), memberList.get(1), message));
Assert.assertEquals(false,
Assert.assertEquals(
false,
message.getReplicable().shouldReplicate(memberList.get(1), memberList.get(0), message));
}
@Test
public void dataReplicateBlackListTest() throws URISyntaxException {
List<LocalMember> memberList = new ArrayList<>();
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8001"),"1"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8002"),"2"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8003"),"3"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8001"), "1"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8002"), "2"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8003"), "3"));
// add node 1 and 2 to the black list
List<LocalMember> blackList = new ArrayList<>();
blackList.add(memberList.get(0));
blackList.add(memberList.get(1));
SharedDataMessage message = getSharedNodeData("blackList", "Disallow some nodes",
new BlackListReplicable<>(blackList));
LocalMember me = getLocalMember(new URI("udp://127.0.0.1:8004"),"4");
SharedDataMessage message =
getSharedNodeData("blackList", "Disallow some nodes", new BlackListReplicable<>(blackList));
LocalMember me = getLocalMember(new URI("udp://127.0.0.1:8004"), "4");
// data should not replicate to node 1 and 2
Assert.assertEquals(false,
message.getReplicable().shouldReplicate(me, memberList.get(0), message));
Assert.assertEquals(false,
message.getReplicable().shouldReplicate(me, memberList.get(1), message));
Assert.assertEquals(true,
message.getReplicable().shouldReplicate(me, memberList.get(2), message));
Assert.assertEquals(
false, message.getReplicable().shouldReplicate(me, memberList.get(0), message));
Assert.assertEquals(
false, message.getReplicable().shouldReplicate(me, memberList.get(1), message));
Assert.assertEquals(
true, message.getReplicable().shouldReplicate(me, memberList.get(2), message));
}
@Test
public void dataReplicateBlackListNullTest() throws URISyntaxException {
List<LocalMember> memberList = new ArrayList<>();
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8001"),"1"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8002"),"2"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8001"), "1"));
memberList.add(getLocalMember(new URI("udp://127.0.0.1:8002"), "2"));
SharedDataMessage message = getSharedNodeData("blackList", "Disallow some nodes",
new BlackListReplicable<>(null));
SharedDataMessage message =
getSharedNodeData("blackList", "Disallow some nodes", new BlackListReplicable<>(null));
// data should replicate if no blacklist specified
Assert.assertEquals(true,
Assert.assertEquals(
true,
message.getReplicable().shouldReplicate(memberList.get(0), memberList.get(1), message));
Assert.assertEquals(true,
Assert.assertEquals(
true,
message.getReplicable().shouldReplicate(memberList.get(1), memberList.get(0), message));
}
@ -157,28 +161,41 @@ public class DataReplicationTest {
List<LocalMember> memberListDc1 = new ArrayList<>();
List<LocalMember> memberListDc2 = new ArrayList<>();
memberListDc1
.add(getLocalMemberDc(new URI("udp://10.0.0.1:8000"), "1", "DataCenter1", "Rack1"));
memberListDc1
.add(getLocalMemberDc(new URI("udp://10.0.0.2:8000"), "2", "DataCenter1", "Rack2"));
memberListDc2
.add(getLocalMemberDc(new URI("udp://10.0.1.1:8000"), "11", "DataCenter2", "Rack1"));
memberListDc2
.add(getLocalMemberDc(new URI("udp://10.0.1.2:8000"), "12", "DataCenter2", "Rack2"));
memberListDc1.add(
getLocalMemberDc(new URI("udp://10.0.0.1:8000"), "1", "DataCenter1", "Rack1"));
memberListDc1.add(
getLocalMemberDc(new URI("udp://10.0.0.2:8000"), "2", "DataCenter1", "Rack2"));
memberListDc2.add(
getLocalMemberDc(new URI("udp://10.0.1.1:8000"), "11", "DataCenter2", "Rack1"));
memberListDc2.add(
getLocalMemberDc(new URI("udp://10.0.1.2:8000"), "12", "DataCenter2", "Rack2"));
SharedDataMessage message = getSharedNodeData("datacenter1", "I am in data center 1 rack 1",
new DataCenterReplicable<>());
SharedDataMessage message =
getSharedNodeData(
"datacenter1", "I am in data center 1 rack 1", new DataCenterReplicable<>());
// data should replicate in data center 1
Assert.assertEquals(true, message.getReplicable()
Assert.assertEquals(
true,
message
.getReplicable()
.shouldReplicate(memberListDc1.get(0), memberListDc1.get(1), message));
Assert.assertEquals(true, message.getReplicable()
Assert.assertEquals(
true,
message
.getReplicable()
.shouldReplicate(memberListDc2.get(0), memberListDc2.get(1), message));
// data should not replicate to data center 2
Assert.assertEquals(false, message.getReplicable()
Assert.assertEquals(
false,
message
.getReplicable()
.shouldReplicate(memberListDc1.get(0), memberListDc2.get(0), message));
Assert.assertEquals(false, message.getReplicable()
Assert.assertEquals(
false,
message
.getReplicable()
.shouldReplicate(memberListDc1.get(1), memberListDc2.get(1), message));
}
@ -186,21 +203,22 @@ public class DataReplicationTest {
public void dataReplicateDataCenterUnknownDataCenterTest() throws URISyntaxException {
List<LocalMember> memberListDc1 = new ArrayList<>();
memberListDc1
.add(getLocalMemberDc(new URI("udp://10.0.0.1:8000"), "1", "DataCenter1", "Rack1"));
memberListDc1.add(
getLocalMemberDc(new URI("udp://10.0.0.1:8000"), "1", "DataCenter1", "Rack1"));
Map<String, String> properties = new HashMap<>();
LocalMember unknownDc = new LocalMember("cluster1", new URI("udp://10.0.1.2:8000"), "12", 0,
properties, 1, 0, "");
LocalMember unknownDc =
new LocalMember("cluster1", new URI("udp://10.0.1.2:8000"), "12", 0, properties, 1, 0, "");
SharedDataMessage message = getSharedNodeData("datacenter1","I am in data center 1 rack 1", new DataCenterReplicable<>());
SharedDataMessage message =
getSharedNodeData(
"datacenter1", "I am in data center 1 rack 1", new DataCenterReplicable<>());
// data should not replicate from dc1 to unknown node
Assert.assertEquals(false, message.getReplicable()
.shouldReplicate(memberListDc1.get(0), unknownDc, message));
Assert.assertEquals(
false, message.getReplicable().shouldReplicate(memberListDc1.get(0), unknownDc, message));
// data can replicate from unknown node to dc
Assert.assertEquals(true, message.getReplicable()
.shouldReplicate(unknownDc, memberListDc1.get(0), message));
Assert.assertEquals(
true, message.getReplicable().shouldReplicate(unknownDc, memberListDc1.get(0), message));
}
}

5
gossip-base/src/test/java/org/apache/gossip/transport/UnitTestTransportManager.java
View File

@ -18,15 +18,14 @@
package org.apache.gossip.transport;
import org.apache.gossip.manager.GossipCore;
import org.apache.gossip.manager.GossipManager;
import java.io.IOException;
import java.net.URI;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.gossip.manager.GossipCore;
import org.apache.gossip.manager.GossipManager;
/** Only use in unit tests! */
public class UnitTestTransportManager extends AbstractTransportManager {

16
gossip-examples/src/main/java/org/apache/gossip/examples/RunStandardExamples.java
View File

@ -21,10 +21,6 @@ import java.io.IOException;
public class RunStandardExamples {
private static boolean WILL_READ = true;
private static boolean WILL_NOT_READ = false;
public static void main(String[] args) {
if ((args.length < 1) || args[0].equals("-h") || args[0].equals("--help") || args.length < 2) {
System.out.print(usage());
@ -37,16 +33,18 @@ public class RunStandardExamples {
System.out.print(usage());
return;
}
runExaple(example, channel);
runExample(example, channel);
} catch (Exception e) {
System.out.print(usage());
}
}
private static void runExaple(int exampleNumber, int channel) throws IOException {
String[] args = stanardArgs(channel, new String[4]);
private static void runExample(int exampleNumber, int channel) throws IOException {
String[] args = standardArgs(channel, new String[4]);
boolean WILL_READ = true;
if (exampleNumber == 1) {
StandAloneNode example = new StandAloneNode(args);
boolean WILL_NOT_READ = false;
example.exec(WILL_NOT_READ);
} else if (exampleNumber == 2) {
StandAloneNodeCrdtOrSet example = new StandAloneNodeCrdtOrSet(args);
@ -61,7 +59,7 @@ public class RunStandardExamples {
}
}
private static String[] stanardArgs(int channel, String[] args) {
private static String[] standardArgs(int channel, String[] args) {
// see README.md for examples
args[0] = "udp://localhost:1000" + channel;
args[1] = "" + channel;
@ -71,7 +69,7 @@ public class RunStandardExamples {
}
private static String[] extendedArgs(int channel, String[] args) {
args = stanardArgs(channel, args);
standardArgs(channel, args);
// see README.md for examples
if (channel == 0) {
args[4] = "1";

6
gossip-examples/src/main/java/org/apache/gossip/examples/StandAloneDatacenterAndRack.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.gossip.GossipSettings;
@ -37,7 +38,7 @@ public class StandAloneDatacenterAndRack extends StandAloneExampleBase {
initGossipManager(args);
}
public static void main(String[] args) throws InterruptedException, IOException {
public static void main(String[] args) throws IOException {
StandAloneDatacenterAndRack example = new StandAloneDatacenterAndRack(args);
boolean willRead = true;
example.exec(willRead);
@ -61,8 +62,7 @@ public class StandAloneDatacenterAndRack extends StandAloneExampleBase {
.uri(URI.create(args[0]))
.id(args[1])
.gossipSettings(s)
.gossipMembers(
Arrays.asList(new RemoteMember("mycluster", URI.create(args[2]), args[3])))
.gossipMembers(List.of(new RemoteMember("mycluster", URI.create(args[2]), args[3])))
.properties(props)
.build();
manager.init();

1
gossip-examples/src/main/java/org/apache/gossip/examples/StandAloneExampleBase.java
View File

@ -23,7 +23,6 @@ import java.io.InputStreamReader;
import java.net.URI;
import java.util.Collections;
import java.util.List;
import org.apache.gossip.GossipSettings;
import org.apache.gossip.LocalMember;
import org.apache.gossip.RemoteMember;

9
gossip-examples/src/main/java/org/apache/gossip/examples/StandAloneNode.java
View File

@ -22,20 +22,17 @@ import org.apache.gossip.manager.GossipManager;
public class StandAloneNode extends StandAloneExampleBase {
private static boolean WILL_READ = false;
StandAloneNode(String[] args) {
args = super.checkArgsForClearFlag(args);
super.initGossipManager(args);
}
public static void main(String[] args) throws InterruptedException, IOException {
public static void main(String[] args) throws IOException {
StandAloneNode example = new StandAloneNode(args);
boolean WILL_READ = false;
example.exec(WILL_READ);
}
@Override
void printValues(GossipManager gossipService) {
}
void printValues(GossipManager gossipService) {}
}

11
gossip-examples/src/main/java/org/apache/gossip/examples/StandAloneNodeCrdtOrSet.java
View File

@ -18,7 +18,6 @@
package org.apache.gossip.examples;
import java.io.IOException;
import org.apache.gossip.crdt.GrowOnlyCounter;
import org.apache.gossip.crdt.OrSet;
import org.apache.gossip.manager.GossipManager;
@ -35,7 +34,7 @@ public class StandAloneNodeCrdtOrSet extends StandAloneExampleBase {
super.initGossipManager(args);
}
public static void main(String[] args) throws InterruptedException, IOException {
public static void main(String[] args) throws IOException {
StandAloneNodeCrdtOrSet example = new StandAloneNodeCrdtOrSet(args);
boolean willRead = true;
example.exec(willRead);
@ -53,7 +52,7 @@ public class StandAloneNodeCrdtOrSet extends StandAloneExampleBase {
private static void gcount(String val, GossipManager gossipManager) {
GrowOnlyCounter c = (GrowOnlyCounter) gossipManager.findCrdt(INDEX_KEY_FOR_COUNTER);
Long l = Long.valueOf(val);
long l = Long.parseLong(val);
if (c == null) {
c = new GrowOnlyCounter(new GrowOnlyCounter.Builder(gossipManager).increment((l)));
} else {
@ -73,7 +72,7 @@ public class StandAloneNodeCrdtOrSet extends StandAloneExampleBase {
SharedDataMessage m = new SharedDataMessage();
m.setExpireAt(Long.MAX_VALUE);
m.setKey(INDEX_KEY_FOR_SET);
m.setPayload(new OrSet<String>(s, new OrSet.Builder<String>().remove(val)));
m.setPayload(new OrSet<>(s, new OrSet.Builder<String>().remove(val)));
m.setTimestamp(System.currentTimeMillis());
gossipService.merge(m);
}
@ -82,7 +81,7 @@ public class StandAloneNodeCrdtOrSet extends StandAloneExampleBase {
SharedDataMessage m = new SharedDataMessage();
m.setExpireAt(Long.MAX_VALUE);
m.setKey(INDEX_KEY_FOR_SET);
m.setPayload(new OrSet<String>(val));
m.setPayload(new OrSet<>(val));
m.setTimestamp(System.currentTimeMillis());
gossipService.merge(m);
}
@ -118,7 +117,7 @@ public class StandAloneNodeCrdtOrSet extends StandAloneExampleBase {
valid = false;
}
} else if (op == 'l') {
if ((val == INDEX_KEY_FOR_SET) || (val == INDEX_KEY_FOR_COUNTER)) {
if ((val.equals(INDEX_KEY_FOR_SET)) || (val.equals(INDEX_KEY_FOR_COUNTER))) {
listen(val, getGossipManager());
} else {
valid = false;

5
gossip-examples/src/main/java/org/apache/gossip/examples/StandAlonePNCounter.java
View File

@ -18,7 +18,6 @@
package org.apache.gossip.examples;
import java.io.IOException;
import org.apache.gossip.crdt.PNCounter;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.model.SharedDataMessage;
@ -30,7 +29,7 @@ public class StandAlonePNCounter extends StandAloneExampleBase {
super.initGossipManager(args);
}
public static void main(String[] args) throws InterruptedException, IOException {
public static void main(String[] args) throws IOException {
StandAlonePNCounter example = new StandAlonePNCounter(args);
boolean willRead = true;
example.exec(willRead);
@ -61,7 +60,7 @@ public class StandAlonePNCounter extends StandAloneExampleBase {
if (valid) {
if (op == 'i') {
increment(l, getGossipManager());
} else if (op == 'd') {
} else {
decrement(l, getGossipManager());
}
}

134
gossip-itest/src/test/java/org/apache/gossip/DataTest.java
View File

@ -52,7 +52,8 @@ public class DataTest {
private final String pnCounterKey = "crdtpn";
@BeforeClass
public static void initializeMembers() throws InterruptedException, UnknownHostException, URISyntaxException{
public static void initializeMembers()
throws InterruptedException, UnknownHostException, URISyntaxException {
final int clusterMembers = 2;
GossipSettings settings = new GossipSettings();
@ -60,107 +61,122 @@ public class DataTest {
settings.setPersistDataState(false);
String cluster = UUID.randomUUID().toString();
List<Member> startupMembers = new ArrayList<>();
for (int i = 0; i < clusterMembers; ++i){
for (int i = 0; i < clusterMembers; ++i) {
int id = i + 1;
URI uri = new URI("udp://" + "127.0.0.1" + ":" + (50000 + id));
startupMembers.add(new RemoteMember(cluster, uri, id + ""));
}
for (Member member : startupMembers){
GossipManager gossipService = GossipManagerBuilder.newBuilder().cluster(cluster).uri(member.getUri())
.id(member.getId()).gossipMembers(startupMembers).gossipSettings(settings).build();
for (Member member : startupMembers) {
GossipManager gossipService =
GossipManagerBuilder.newBuilder()
.cluster(cluster)
.uri(member.getUri())
.id(member.getId())
.gossipMembers(startupMembers)
.gossipSettings(settings)
.build();
clients.add(gossipService);
gossipService.init();
}
}
@AfterClass
public static void shutdownMembers(){
for (final GossipManager client : clients){
public static void shutdownMembers() {
for (final GossipManager client : clients) {
client.shutdown();
}
}
@Test
public void simpleDataTest(){
TUnit.assertThat(() -> {
public void simpleDataTest() {
TUnit.assertThat(
() -> {
int total = 0;
for (GossipManager client : clients){
for (GossipManager client : clients) {
total += client.getLiveMembers().size();
}
return total;
}).afterWaitingAtMost(10, TimeUnit.SECONDS).isEqualTo(2);
})
.afterWaitingAtMost(10, TimeUnit.SECONDS)
.isEqualTo(2);
clients.get(0).gossipPerNodeData(generatePerNodeMsg("a", "b"));
clients.get(0).gossipSharedData(generateSharedMsg("a", "c"));
TUnit.assertThat(() -> {
TUnit.assertThat(
() -> {
PerNodeDataMessage x = clients.get(1).findPerNodeGossipData(1 + "", "a");
if (x == null)
return "";
else
return x.getPayload();
}).afterWaitingAtMost(10, TimeUnit.SECONDS).isEqualTo("b");
if (x == null) return "";
else return x.getPayload();
})
.afterWaitingAtMost(10, TimeUnit.SECONDS)
.isEqualTo("b");
TUnit.assertThat(() -> {
TUnit.assertThat(
() -> {
SharedDataMessage x = clients.get(1).findSharedGossipData("a");
if (x == null)
return "";
else
return x.getPayload();
}).afterWaitingAtMost(10, TimeUnit.SECONDS).isEqualTo("c");
if (x == null) return "";
else return x.getPayload();
})
.afterWaitingAtMost(10, TimeUnit.SECONDS)
.isEqualTo("c");
}
Set<String> setFromList(String... elements){
Set<String> setFromList(String... elements) {
return new HashSet<>(Arrays.asList(elements));
}
void crdtSetTest(String key, Function<Set<String>, CrdtAddRemoveSet<String, Set<String>, ?>> construct){
//populate
void crdtSetTest(
String key, Function<Set<String>, CrdtAddRemoveSet<String, Set<String>, ?>> construct) {
// populate
clients.get(0).merge(generateSharedMsg(key, construct.apply(setFromList("1", "2"))));
clients.get(1).merge(generateSharedMsg(key, construct.apply(setFromList("3", "4"))));
assertMergedCrdt(key, construct.apply(setFromList("1", "2", "3", "4")).value());
//drop element
// drop element
@SuppressWarnings("unchecked")
CrdtAddRemoveSet<String, ?, ?> set = (CrdtAddRemoveSet<String, ?, ?>) clients.get(0).findCrdt(key);
CrdtAddRemoveSet<String, ?, ?> set =
(CrdtAddRemoveSet<String, ?, ?>) clients.get(0).findCrdt(key);
clients.get(0).merge(generateSharedMsg(key, set.remove("3")));
//assert deletion
// assert deletion
assertMergedCrdt(key, construct.apply(setFromList("1", "2", "4")).value());
}
@Test
public void OrSetTest(){
public void OrSetTest() {
crdtSetTest("cror", OrSet::new);
}
@Test
public void LWWSetTest(){
public void LWWSetTest() {
crdtSetTest("crlww", LwwSet::new);
}
@Test
public void MaxChangeSetTest(){
public void MaxChangeSetTest() {
crdtSetTest("crmcs", MaxChangeSet::new);
}
@Test
public void TwoPhaseSetTest(){
public void TwoPhaseSetTest() {
crdtSetTest("crtps", TwoPhaseSet::new);
}
@Test
public void GrowOnlyCounterTest(){
Consumer<Long> assertCountUpdated = count -> {
for (GossipManager client : clients){
public void GrowOnlyCounterTest() {
Consumer<Long> assertCountUpdated =
count -> {
for (GossipManager client : clients) {
TUnit.assertThat(() -> client.findCrdt(gCounterKey))
.afterWaitingAtMost(10, TimeUnit.SECONDS)
.isEqualTo(new GrowOnlyCounter(new GrowOnlyCounter.Builder(client).increment(count)));
.isEqualTo(
new GrowOnlyCounter(new GrowOnlyCounter.Builder(client).increment(count)));
}
};
//generate different increment
// generate different increment
Object payload = new GrowOnlyCounter(new GrowOnlyCounter.Builder(clients.get(0)).increment(1L));
clients.get(0).merge(generateSharedMsg(gCounterKey, payload));
payload = new GrowOnlyCounter(new GrowOnlyCounter.Builder(clients.get(1)).increment(2L));
@ -168,30 +184,39 @@ public class DataTest {
assertCountUpdated.accept((long) 3);
//update one
// update one
GrowOnlyCounter gc = (GrowOnlyCounter) clients.get(1).findCrdt(gCounterKey);
GrowOnlyCounter gc2 = new GrowOnlyCounter(gc,
new GrowOnlyCounter.Builder(clients.get(1)).increment(4L));
GrowOnlyCounter gc2 =
new GrowOnlyCounter(gc, new GrowOnlyCounter.Builder(clients.get(1)).increment(4L));
clients.get(1).merge(generateSharedMsg(gCounterKey, gc2));
assertCountUpdated.accept((long) 7);
}
@Test
public void PNCounterTest(){
Consumer<List<Integer>> counterUpdate = list -> {
public void PNCounterTest() {
Consumer<List<Integer>> counterUpdate =
list -> {
int clientIndex = 0;
for (int delta : list){
for (int delta : list) {
PNCounter c = (PNCounter) clients.get(clientIndex).findCrdt(pnCounterKey);
c = new PNCounter(c, new PNCounter.Builder(clients.get(clientIndex)).increment(((long) delta)));
c =
new PNCounter(
c, new PNCounter.Builder(clients.get(clientIndex)).increment(((long) delta)));
clients.get(clientIndex).merge(generateSharedMsg(pnCounterKey, c));
clientIndex = (clientIndex + 1) % clients.size();
}
};
// given PNCounter
clients.get(0).merge(generateSharedMsg(pnCounterKey, new PNCounter(new PNCounter.Builder(clients.get(0)))));
clients.get(1).merge(generateSharedMsg(pnCounterKey, new PNCounter(new PNCounter.Builder(clients.get(1)))));
clients
.get(0)
.merge(
generateSharedMsg(pnCounterKey, new PNCounter(new PNCounter.Builder(clients.get(0)))));
clients
.get(1)
.merge(
generateSharedMsg(pnCounterKey, new PNCounter(new PNCounter.Builder(clients.get(1)))));
assertMergedCrdt(pnCounterKey, (long) 0);
@ -203,28 +228,29 @@ public class DataTest {
Long[] expectedResults = {5L, 7L, 9L, 3L};
for (int i = 0; i < updateLists.size(); i++){
for (int i = 0; i < updateLists.size(); i++) {
counterUpdate.accept(updateLists.get(i));
assertMergedCrdt(pnCounterKey, expectedResults[i]);
}
}
@Test
public void GrowOnlySetTest(){
public void GrowOnlySetTest() {
clients.get(0).merge(generateSharedMsg("cr", new GrowOnlySet<>(Arrays.asList("1"))));
clients.get(1).merge(generateSharedMsg("cr", new GrowOnlySet<>(Arrays.asList("2"))));
assertMergedCrdt("cr", new GrowOnlySet<>(Arrays.asList("1", "2")).value());
}
private void assertMergedCrdt(String key, Object expected){
for (GossipManager client : clients){
private void assertMergedCrdt(String key, Object expected) {
for (GossipManager client : clients) {
TUnit.assertThat(() -> client.findCrdt(key).value())
.afterWaitingAtMost(10, TimeUnit.SECONDS).isEqualTo(expected);
.afterWaitingAtMost(10, TimeUnit.SECONDS)
.isEqualTo(expected);
}
}
private PerNodeDataMessage generatePerNodeMsg(String key, Object payload){
private PerNodeDataMessage generatePerNodeMsg(String key, Object payload) {
PerNodeDataMessage g = new PerNodeDataMessage();
g.setExpireAt(Long.MAX_VALUE);
g.setKey(key);
@ -233,7 +259,7 @@ public class DataTest {
return g;
}
private SharedDataMessage generateSharedMsg(String key, Object payload){
private SharedDataMessage generateSharedMsg(String key, Object payload) {
SharedDataMessage d = new SharedDataMessage();
d.setKey(key);
d.setPayload(payload);

4
gossip-itest/src/test/java/org/apache/gossip/IdAndPropertyTest.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.gossip;
import io.teknek.tunit.TUnit;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
@ -26,14 +27,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.gossip.manager.DatacenterRackAwareActiveGossiper;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
import org.junit.jupiter.api.Test;
import io.teknek.tunit.TUnit;
public class IdAndPropertyTest extends AbstractIntegrationBase {
@Test

13
gossip-itest/src/test/java/org/apache/gossip/PerNodeDataEventTest.java
View File

@ -18,6 +18,12 @@
package org.apache.gossip;
import io.teknek.tunit.TUnit;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
import org.apache.gossip.model.PerNodeDataMessage;
@ -26,13 +32,6 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
@RunWith(Parameterized.class)
public class PerNodeDataEventTest extends AbstractIntegrationBase {

11
gossip-itest/src/test/java/org/apache/gossip/PerNodeDataReplicationControlTest.java
View File

@ -18,6 +18,11 @@
package org.apache.gossip;
import io.teknek.tunit.TUnit;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import org.apache.gossip.manager.DatacenterRackAwareActiveGossiper;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
@ -25,12 +30,6 @@ import org.apache.gossip.model.PerNodeDataMessage;
import org.apache.gossip.replication.*;
import org.junit.Test;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.TimeUnit;
public class PerNodeDataReplicationControlTest extends AbstractIntegrationBase {
@Test

66
gossip-itest/src/test/java/org/apache/gossip/SharedDataEventTest.java
View File

@ -51,9 +51,7 @@ public class SharedDataEventTest extends AbstractIntegrationBase {
@Parameterized.Parameters(name = "{index} bulkTransfer={1}")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][]{
{50000, false}, {55000, true}
});
return Arrays.asList(new Object[][] {{50000, false}, {55000, true}});
}
@Test
@ -74,29 +72,40 @@ public class SharedDataEventTest extends AbstractIntegrationBase {
final int clusterMembers = 2;
for (int i = 1; i < clusterMembers + 1; ++i) {
URI uri = new URI("udp://" + "127.0.0.1" + ":" + (base + i));
GossipManager gossipService = GossipManagerBuilder.newBuilder().cluster(cluster).uri(uri)
.id(i + "").gossipMembers(startupMembers).gossipSettings(settings).build();
GossipManager gossipService =
GossipManagerBuilder.newBuilder()
.cluster(cluster)
.uri(uri)
.id(i + "")
.gossipMembers(startupMembers)
.gossipSettings(settings)
.build();
clients.add(gossipService);
gossipService.init();
register(gossipService);
}
// check whether the members are discovered
TUnit.assertThat(() -> {
TUnit.assertThat(
() -> {
int total = 0;
for (int i = 0; i < clusterMembers; ++i) {
total += clients.get(i).getLiveMembers().size();
}
return total;
}).afterWaitingAtMost(20, TimeUnit.SECONDS).isEqualTo(2);
})
.afterWaitingAtMost(20, TimeUnit.SECONDS)
.isEqualTo(2);
// Adding new data to Node 1
clients.get(0).gossipSharedData(sharedNodeData("category", "distributed"));
// Node 2 is interested in data changes for the key "organization" and "category"
clients.get(1).registerSharedDataSubscriber((key, oldValue, newValue) -> {
if (!key.equals("organization") && !key.equals("category"))
return;
clients
.get(1)
.registerSharedDataSubscriber(
(key, oldValue, newValue) -> {
if (!key.equals("organization") && !key.equals("category")) return;
receivedKey = key;
receivingNodeDataOldValue = oldValue;
receivingNodeDataNewValue = newValue;
@ -125,7 +134,6 @@ public class SharedDataEventTest extends AbstractIntegrationBase {
Assert.assertEquals("organization", receivedKey);
Assert.assertEquals("apache", receivingNodeDataOldValue);
Assert.assertEquals("apache-gossip", receivingNodeDataNewValue);
}
@Test
@ -145,23 +153,35 @@ public class SharedDataEventTest extends AbstractIntegrationBase {
final int clusterMembers = 3;
for (int i = 1; i < clusterMembers + 1; ++i) {
URI uri = new URI("udp://" + "127.0.0.1" + ":" + (base + i));
GossipManager gossipService = GossipManagerBuilder.newBuilder().cluster(cluster).uri(uri)
.id(i + "").gossipMembers(startupMembers).gossipSettings(settings).build();
GossipManager gossipService =
GossipManagerBuilder.newBuilder()
.cluster(cluster)
.uri(uri)
.id(i + "")
.gossipMembers(startupMembers)
.gossipSettings(settings)
.build();
clients.add(gossipService);
gossipService.init();
register(gossipService);
}
// check whether the members are discovered
TUnit.assertThat(() -> {
TUnit.assertThat(
() -> {
int total = 0;
for (int i = 0; i < clusterMembers; ++i) {
total += clients.get(i).getLiveMembers().size();
}
return total;
}).afterWaitingAtMost(20, TimeUnit.SECONDS).isEqualTo(2);
})
.afterWaitingAtMost(20, TimeUnit.SECONDS)
.isEqualTo(2);
clients.get(1).registerSharedDataSubscriber((key, oldValue, newValue) -> {
clients
.get(1)
.registerSharedDataSubscriber(
(key, oldValue, newValue) -> {
receivedKey = key;
receivingNodeDataOldValue = oldValue;
receivingNodeDataNewValue = newValue;
@ -184,20 +204,22 @@ public class SharedDataEventTest extends AbstractIntegrationBase {
Assert.assertEquals(1, ((GrowOnlyCounter) receivingNodeDataNewValue).value().longValue());
// check whether Node 3 received the gCounter
TUnit.assertThat(() -> {
TUnit.assertThat(
() -> {
GrowOnlyCounter gc = (GrowOnlyCounter) clients.get(2).findCrdt(gCounterKey);
if (gc == null) {
return "";
} else {
return gc;
}
}).afterWaitingAtMost(10, TimeUnit.SECONDS).isEqualTo(
new GrowOnlyCounter(new GrowOnlyCounter.Builder(clients.get(0)).increment(1L)));
})
.afterWaitingAtMost(10, TimeUnit.SECONDS)
.isEqualTo(new GrowOnlyCounter(new GrowOnlyCounter.Builder(clients.get(0)).increment(1L)));
// Node 3 Updates the gCounter by 4
GrowOnlyCounter gc = (GrowOnlyCounter) clients.get(2).findCrdt(gCounterKey);
GrowOnlyCounter gcNew = new GrowOnlyCounter(gc,
new GrowOnlyCounter.Builder(clients.get(2)).increment(4L));
GrowOnlyCounter gcNew =
new GrowOnlyCounter(gc, new GrowOnlyCounter.Builder(clients.get(2)).increment(4L));
d = new SharedDataMessage();
d.setKey(gCounterKey);
@ -213,7 +235,6 @@ public class SharedDataEventTest extends AbstractIntegrationBase {
Assert.assertEquals(1, ((GrowOnlyCounter) receivingNodeDataOldValue).value().longValue());
Assert.assertTrue(receivingNodeDataNewValue instanceof GrowOnlyCounter);
Assert.assertEquals(5, ((GrowOnlyCounter) receivingNodeDataNewValue).value().longValue());
}
private SharedDataMessage sharedNodeData(String key, String value) {
@ -224,5 +245,4 @@ public class SharedDataEventTest extends AbstractIntegrationBase {
g.setTimestamp(System.currentTimeMillis());
return g;
}
}

13
gossip-itest/src/test/java/org/apache/gossip/SharedDataLockTest.java
View File

@ -17,13 +17,6 @@
*/
package org.apache.gossip;
import org.apache.gossip.lock.exceptions.VoteFailedException;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
import org.apache.gossip.model.SharedDataMessage;
import org.junit.Assert;
import org.junit.Test;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
@ -31,6 +24,12 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.gossip.lock.exceptions.VoteFailedException;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
import org.apache.gossip.model.SharedDataMessage;
import org.junit.Assert;
import org.junit.Test;
public class SharedDataLockTest extends AbstractIntegrationBase {

3
gossip-itest/src/test/java/org/apache/gossip/SharedDataReplicationControlTest.java
View File

@ -44,8 +44,7 @@ import java.util.concurrent.TimeUnit;
public class SharedDataReplicationControlTest extends AbstractIntegrationBase {
@Test
public void sharedDataReplicationTest()
throws URISyntaxException {
public void sharedDataReplicationTest() throws URISyntaxException {
generateStandardNodes(3);
// check whether the members are discovered

3
gossip-itest/src/test/java/org/apache/gossip/ShutdownDeadtimeTest.java
View File

@ -38,8 +38,7 @@ public class ShutdownDeadtimeTest {
// harm), and the
// sleep that happens after startup.
@Test
public void DeadNodesDoNotComeAliveAgain()
throws InterruptedException, URISyntaxException {
public void DeadNodesDoNotComeAliveAgain() throws InterruptedException, URISyntaxException {
GossipSettings settings = new GossipSettings(100, 10000, 1000, 1, 10.0, "normal", false);
settings.setPersistRingState(false);
settings.setPersistDataState(false);

4
gossip-itest/src/test/java/org/apache/gossip/SignedMessageTest.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.gossip;
import io.teknek.tunit.TUnit;
import java.io.File;
import java.io.IOException;
import java.net.URI;
@ -28,7 +29,6 @@ import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
import org.apache.gossip.manager.PassiveGossipConstants;
@ -36,8 +36,6 @@ import org.apache.gossip.secure.KeyTool;
import org.junit.Assert;
import org.junit.Test;
import io.teknek.tunit.TUnit;
public class SignedMessageTest extends AbstractIntegrationBase {
private GossipSettings gossiperThatSigns() {

10
gossip-itest/src/test/java/org/apache/gossip/StartupSettingsTest.java
View File

@ -17,18 +17,16 @@
*/
package org.apache.gossip;
import lombok.extern.slf4j.Slf4j;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.manager.GossipManagerBuilder;
import org.junit.jupiter.api.Test;
/** Tests support of using {@code StartupSettings} and thereby reading setup config from file. */
@Slf4j

1
gossip-itest/src/test/java/org/apache/gossip/TenNodeThreeSeedTest.java
View File

@ -18,7 +18,6 @@
package org.apache.gossip;
import io.teknek.tunit.TUnit;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;

13
gossip-protocol-jackson/src/main/java/org/apache/gossip/protocol/json/JacksonProtocolManager.java
View File

@ -21,13 +21,6 @@ import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.gossip.GossipSettings;
import org.apache.gossip.crdt.CrdtModule;
import org.apache.gossip.manager.PassiveGossipConstants;
import org.apache.gossip.model.Base;
import org.apache.gossip.model.SignedPayload;
import org.apache.gossip.protocol.ProtocolManager;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -40,6 +33,12 @@ import java.security.Signature;
import java.security.SignatureException;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import org.apache.gossip.GossipSettings;
import org.apache.gossip.crdt.CrdtModule;
import org.apache.gossip.manager.PassiveGossipConstants;
import org.apache.gossip.model.Base;
import org.apache.gossip.model.SignedPayload;
import org.apache.gossip.protocol.ProtocolManager;
// this class is constructed by reflection in GossipManager.
public class JacksonProtocolManager implements ProtocolManager {

4
gossip-protocol-jackson/src/test/java/org/apache/gossip/protocol/json/JacksonTest.java
View File

@ -18,6 +18,8 @@
package org.apache.gossip.protocol.json;
import static org.junit.jupiter.api.Assertions.*;
import com.codahale.metrics.MetricRegistry;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
@ -38,8 +40,6 @@ import org.apache.gossip.protocol.ProtocolManager;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
public class JacksonTest {
private static GossipSettings simpleSettings(GossipSettings settings) {

8
gossip-protocol-jackson/src/test/java/org/apache/gossip/protocol/json/TestMessage.java
View File

@ -17,15 +17,13 @@
*/
package org.apache.gossip.protocol.json;
import lombok.Data;
import lombok.Getter;
import org.apache.gossip.model.Base;
import org.apache.gossip.udp.Trackable;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import lombok.Data;
import org.apache.gossip.model.Base;
import org.apache.gossip.udp.Trackable;
/*
* Here is a test class for serialization. I've tried to include a lot of things in it including nested classes.

11
gossip-transport-udp/src/main/java/org/apache/gossip/transport/udp/UdpTransportManager.java
View File

@ -17,12 +17,6 @@
*/
package org.apache.gossip.transport.udp;
import lombok.extern.slf4j.Slf4j;
import org.apache.gossip.manager.GossipCore;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.model.Base;
import org.apache.gossip.transport.AbstractTransportManager;
import java.io.IOException;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
@ -32,6 +26,11 @@ import java.net.SocketAddress;
import java.net.SocketException;
import java.net.URI;
import java.util.concurrent.atomic.AtomicBoolean;
import lombok.extern.slf4j.Slf4j;
import org.apache.gossip.manager.GossipCore;
import org.apache.gossip.manager.GossipManager;
import org.apache.gossip.model.Base;
import org.apache.gossip.transport.AbstractTransportManager;
/**
* This class is constructed by reflection in GossipManager. It manages transport (byte read/write)