GOSSIP-81 Move Jackson and UDP to their own modules
Part of what makes this work is the test implementation of TransportManager. This PR is pretty straightforward. A few gotchas though: * A message signing test was moved into `JacksonTests` because that is where the signing actually happens. * A CRDT serializing test was moved there as well. It's the best place for now. * No UDP tests at all. I plan to fix that in a bit. Reasoning is that it is difficult to test any TransportManager implementation without bring up a full stack. I plan to address this in the future (GOSSIP-83). * Simple round trip Jackson serialization tests.
This commit is contained in:
Gary Dusbabek
@ -0,0 +1,132 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.gossip.protocol.json;
|
||||
|
||||
import com.codahale.metrics.Meter;
|
||||
import com.codahale.metrics.MetricRegistry;
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.gossip.GossipSettings;
|
||||
import org.apache.gossip.crdt.CrdtModule;
|
||||
import org.apache.gossip.manager.PassiveGossipConstants;
|
||||
import org.apache.gossip.model.Base;
|
||||
import org.apache.gossip.model.SignedPayload;
|
||||
import org.apache.gossip.protocol.ProtocolManager;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.security.InvalidKeyException;
|
||||
import java.security.KeyFactory;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.NoSuchProviderException;
|
||||
import java.security.PrivateKey;
|
||||
import java.security.Signature;
|
||||
import java.security.SignatureException;
|
||||
import java.security.spec.InvalidKeySpecException;
|
||||
import java.security.spec.PKCS8EncodedKeySpec;
|
||||
|
||||
// this class is constructed by reflection in GossipManager.
|
||||
public class JacksonProtocolManager implements ProtocolManager {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final PrivateKey privKey;
|
||||
private final Meter signed;
|
||||
private final Meter unsigned;
|
||||
|
||||
/** required for reflection to work! */
|
||||
public JacksonProtocolManager(GossipSettings settings, String id, MetricRegistry registry) {
|
||||
// set up object mapper.
|
||||
objectMapper = buildObjectMapper(settings);
|
||||
|
||||
// set up message signing.
|
||||
if (settings.isSignMessages()){
|
||||
File privateKey = new File(settings.getPathToKeyStore(), id);
|
||||
File publicKey = new File(settings.getPathToKeyStore(), id + ".pub");
|
||||
if (!privateKey.exists()){
|
||||
throw new IllegalArgumentException("private key not found " + privateKey);
|
||||
}
|
||||
if (!publicKey.exists()){
|
||||
throw new IllegalArgumentException("public key not found " + publicKey);
|
||||
}
|
||||
try (FileInputStream keyfis = new FileInputStream(privateKey)) {
|
||||
byte[] encKey = new byte[keyfis.available()];
|
||||
keyfis.read(encKey);
|
||||
keyfis.close();
|
||||
PKCS8EncodedKeySpec privKeySpec = new PKCS8EncodedKeySpec(encKey);
|
||||
KeyFactory keyFactory = KeyFactory.getInstance("DSA");
|
||||
privKey = keyFactory.generatePrivate(privKeySpec);
|
||||
} catch (NoSuchAlgorithmException | InvalidKeySpecException | IOException e) {
|
||||
throw new RuntimeException("failed hard", e);
|
||||
}
|
||||
} else {
|
||||
privKey = null;
|
||||
}
|
||||
|
||||
signed = registry.meter(PassiveGossipConstants.SIGNED_MESSAGE);
|
||||
unsigned = registry.meter(PassiveGossipConstants.UNSIGNED_MESSAGE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] write(Base message) throws IOException {
|
||||
byte[] json_bytes;
|
||||
if (privKey == null){
|
||||
json_bytes = objectMapper.writeValueAsBytes(message);
|
||||
} else {
|
||||
SignedPayload p = new SignedPayload();
|
||||
p.setData(objectMapper.writeValueAsString(message).getBytes());
|
||||
p.setSignature(sign(p.getData(), privKey));
|
||||
json_bytes = objectMapper.writeValueAsBytes(p);
|
||||
}
|
||||
return json_bytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Base read(byte[] buf) throws IOException {
|
||||
Base activeGossipMessage = objectMapper.readValue(buf, Base.class);
|
||||
if (activeGossipMessage instanceof SignedPayload){
|
||||
SignedPayload s = (SignedPayload) activeGossipMessage;
|
||||
signed.mark();
|
||||
return objectMapper.readValue(s.getData(), Base.class);
|
||||
} else {
|
||||
unsigned.mark();
|
||||
return activeGossipMessage;
|
||||
}
|
||||
}
|
||||
|
||||
public static ObjectMapper buildObjectMapper(GossipSettings settings) {
|
||||
ObjectMapper om = new ObjectMapper();
|
||||
om.enableDefaultTyping();
|
||||
// todo: should be specified in the configuration.
|
||||
om.registerModule(new CrdtModule());
|
||||
om.configure(JsonGenerator.Feature.WRITE_NUMBERS_AS_STRINGS, false);
|
||||
return om;
|
||||
}
|
||||
|
||||
private static byte[] sign(byte [] bytes, PrivateKey pk){
|
||||
Signature dsa;
|
||||
try {
|
||||
dsa = Signature.getInstance("SHA1withDSA", "SUN");
|
||||
dsa.initSign(pk);
|
||||
dsa.update(bytes);
|
||||
return dsa.sign();
|
||||
} catch (NoSuchAlgorithmException | NoSuchProviderException | InvalidKeyException | SignatureException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,120 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.gossip.protocol.json;
|
||||
|
||||
import com.codahale.metrics.MetricRegistry;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.gossip.GossipSettings;
|
||||
import org.apache.gossip.Member;
|
||||
import org.apache.gossip.crdt.OrSet;
|
||||
import org.apache.gossip.manager.GossipManager;
|
||||
import org.apache.gossip.manager.GossipManagerBuilder;
|
||||
import org.apache.gossip.model.Base;
|
||||
import org.apache.gossip.protocol.ProtocolManager;
|
||||
import org.apache.gossip.udp.Trackable;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
|
||||
public class JacksonTest {
|
||||
|
||||
private static GossipSettings simpleSettings(GossipSettings settings) {
|
||||
settings.setPersistRingState(false);
|
||||
settings.setPersistDataState(false);
|
||||
settings.setTransportManagerClass("org.apache.gossip.transport.UnitTestTransportManager");
|
||||
settings.setProtocolManagerClass("org.apache.gossip.protocol.json.JacksonProtocolManager");
|
||||
return settings;
|
||||
}
|
||||
|
||||
private static GossipSettings withSigning(GossipSettings settings) {
|
||||
settings.setSignMessages(true);
|
||||
return settings;
|
||||
}
|
||||
|
||||
// formerly of SignedMessageTest.
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void ifSignMustHaveKeys()
|
||||
throws URISyntaxException, UnknownHostException, InterruptedException {
|
||||
String cluster = UUID.randomUUID().toString();
|
||||
GossipSettings settings = withSigning(simpleSettings(new GossipSettings()));
|
||||
List<Member> startupMembers = new ArrayList<>();
|
||||
URI uri = new URI("udp://" + "127.0.0.1" + ":" + (30000 + 1));
|
||||
GossipManager gossipService = GossipManagerBuilder.newBuilder()
|
||||
.cluster(cluster)
|
||||
.uri(uri)
|
||||
.id(1 + "")
|
||||
.gossipMembers(startupMembers)
|
||||
.gossipSettings(settings)
|
||||
.build();
|
||||
gossipService.init();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void jacksonSerialTest() throws InterruptedException, URISyntaxException, IOException {
|
||||
ObjectMapper objectMapper = JacksonProtocolManager.buildObjectMapper(simpleSettings(new GossipSettings()));
|
||||
|
||||
OrSet<Integer> i = new OrSet<Integer>(new OrSet.Builder<Integer>().add(1).remove(1));
|
||||
String s = objectMapper.writeValueAsString(i);
|
||||
@SuppressWarnings("unchecked")
|
||||
OrSet<Integer> back = objectMapper.readValue(s, OrSet.class);
|
||||
Assert.assertEquals(back, i);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMessageEqualityAssumptions() {
|
||||
long timeA = System.nanoTime();
|
||||
long timeB = System.nanoTime();
|
||||
Assert.assertNotEquals(timeA, timeB);
|
||||
|
||||
TestMessage messageA0 = new TestMessage(Long.toHexString(timeA));
|
||||
TestMessage messageA1 = new TestMessage(Long.toHexString(timeA));
|
||||
TestMessage messageB = new TestMessage(Long.toHexString(timeB));
|
||||
|
||||
Assert.assertEquals(messageA0, messageA1);
|
||||
Assert.assertFalse(messageA0 == messageA1);
|
||||
Assert.assertNotEquals(messageA0, messageB);
|
||||
Assert.assertNotEquals(messageA1, messageB);
|
||||
}
|
||||
|
||||
// ideally, we would test the serializability of every message type, but we just want to make sure this works in
|
||||
// basic cases.
|
||||
@Test
|
||||
public void testMessageSerializationRoundTrip() throws Exception {
|
||||
ProtocolManager mgr = new JacksonProtocolManager(simpleSettings(new GossipSettings()), "foo", new MetricRegistry());
|
||||
for (int i = 0; i < 100; i++) {
|
||||
TestMessage a = new TestMessage(Long.toHexString(System.nanoTime()));
|
||||
byte[] bytes = mgr.write(a);
|
||||
TestMessage b = (TestMessage) mgr.read(bytes);
|
||||
Assert.assertFalse(a == b);
|
||||
Assert.assertEquals(a, b);
|
||||
Assert.assertEquals(a.getMapOfThings(), b.getMapOfThings()); // concerned about that one, so explicit check.
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,199 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.gossip.protocol.json;
|
||||
|
||||
import org.apache.gossip.model.Base;
|
||||
import org.apache.gossip.udp.Trackable;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/*
|
||||
* Here is a test class for serialization. I've tried to include a lot of things in it including nested classes.
|
||||
* Note that there are no Jackson annotations.
|
||||
* getters and setters are the keys to making this work without the Jackson annotations.
|
||||
*/
|
||||
class TestMessage extends Base implements Trackable {
|
||||
private String unique;
|
||||
private String from;
|
||||
private String uuid;
|
||||
private String derivedField;
|
||||
private Subclass otherThing;
|
||||
private float floatValue;
|
||||
private double doubleValue;
|
||||
private Object[] arrayOfThings;
|
||||
private Map<String, String> mapOfThings = new HashMap<>();
|
||||
|
||||
private TestMessage() {
|
||||
}
|
||||
|
||||
TestMessage(String unique) {
|
||||
this.unique = unique;
|
||||
from = Integer.toHexString(unique.hashCode());
|
||||
uuid = Integer.toHexString(from.hashCode());
|
||||
derivedField = Integer.toHexString(uuid.hashCode());
|
||||
otherThing = new Subclass(Integer.toHexString(derivedField.hashCode()));
|
||||
floatValue = (float) unique.hashCode() / (float) from.hashCode();
|
||||
doubleValue = (double) uuid.hashCode() / (double) derivedField.hashCode();
|
||||
arrayOfThings = new Object[]{
|
||||
this.unique, from, uuid, derivedField, otherThing, floatValue, doubleValue
|
||||
};
|
||||
|
||||
String curThing = unique;
|
||||
for (int i = 0; i < 100; i++) {
|
||||
String key = Integer.toHexString(curThing.hashCode());
|
||||
String value = Integer.toHexString(key.hashCode());
|
||||
curThing = value;
|
||||
mapOfThings.put(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUriFrom() {
|
||||
return from;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setUriFrom(String uriFrom) {
|
||||
this.from = uriFrom;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUuid() {
|
||||
return uuid;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setUuid(String uuid) {
|
||||
this.uuid = uuid;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (!(o instanceof TestMessage)) return false;
|
||||
TestMessage that = (TestMessage) o;
|
||||
return Objects.equals(unique, that.unique) &&
|
||||
Objects.equals(from, that.from) &&
|
||||
Objects.equals(getUuid(), that.getUuid()) &&
|
||||
Objects.equals(derivedField, that.derivedField) &&
|
||||
Objects.equals(floatValue, that.floatValue) &&
|
||||
Objects.equals(doubleValue, that.doubleValue) &&
|
||||
Arrays.equals(arrayOfThings, that.arrayOfThings) &&
|
||||
Objects.equals(mapOfThings, that.mapOfThings);
|
||||
}
|
||||
|
||||
public String getUnique() {
|
||||
return unique;
|
||||
}
|
||||
|
||||
public void setUnique(String unique) {
|
||||
this.unique = unique;
|
||||
}
|
||||
|
||||
public String getFrom() {
|
||||
return from;
|
||||
}
|
||||
|
||||
public void setFrom(String from) {
|
||||
this.from = from;
|
||||
}
|
||||
|
||||
public String getDerivedField() {
|
||||
return derivedField;
|
||||
}
|
||||
|
||||
public void setDerivedField(String derivedField) {
|
||||
this.derivedField = derivedField;
|
||||
}
|
||||
|
||||
public Subclass getOtherThing() {
|
||||
return otherThing;
|
||||
}
|
||||
|
||||
public void setOtherThing(Subclass otherThing) {
|
||||
this.otherThing = otherThing;
|
||||
}
|
||||
|
||||
public float getFloatValue() {
|
||||
return floatValue;
|
||||
}
|
||||
|
||||
public void setFloatValue(float floatValue) {
|
||||
this.floatValue = floatValue;
|
||||
}
|
||||
|
||||
public double getDoubleValue() {
|
||||
return doubleValue;
|
||||
}
|
||||
|
||||
public void setDoubleValue(double doubleValue) {
|
||||
this.doubleValue = doubleValue;
|
||||
}
|
||||
|
||||
public Object[] getArrayOfThings() {
|
||||
return arrayOfThings;
|
||||
}
|
||||
|
||||
public void setArrayOfThings(Object[] arrayOfThings) {
|
||||
this.arrayOfThings = arrayOfThings;
|
||||
}
|
||||
|
||||
public Map<String, String> getMapOfThings() {
|
||||
return mapOfThings;
|
||||
}
|
||||
|
||||
public void setMapOfThings(Map<String, String> mapOfThings) {
|
||||
this.mapOfThings = mapOfThings;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(unique, getUriFrom(), getUuid(), derivedField, floatValue, doubleValue, arrayOfThings, mapOfThings);
|
||||
}
|
||||
|
||||
static class Subclass {
|
||||
private String thing;
|
||||
|
||||
public Subclass() {
|
||||
}
|
||||
|
||||
public Subclass(String thing) {
|
||||
this.thing = thing;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (!(o instanceof Subclass)) return false;
|
||||
Subclass subclass = (Subclass) o;
|
||||
return Objects.equals(thing, subclass.thing);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(thing);
|
||||
}
|
||||
|
||||
public String getThing() {
|
||||
return thing;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user