Compare commits

..

No commits in common. "main" and "couchdb-version" have entirely different histories.

10 changed files with 48 additions and 202 deletions

31
pom.xml
View file

@ -12,7 +12,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id> <quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
<quarkus.platform.group-id>io.quarkus.platform</quarkus.platform.group-id> <quarkus.platform.group-id>io.quarkus.platform</quarkus.platform.group-id>
<quarkus.platform.version>2.15.1.Final</quarkus.platform.version> <quarkus.platform.version>2.13.1.Final</quarkus.platform.version>
<skipITs>true</skipITs> <skipITs>true</skipITs>
<surefire-plugin.version>3.0.0-M7</surefire-plugin.version> <surefire-plugin.version>3.0.0-M7</surefire-plugin.version>
</properties> </properties>
@ -35,10 +35,10 @@
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
<dependencies> <dependencies>
<dependency> <!-- <dependency>-->
<groupId>org.apache.camel.quarkus</groupId> <!-- <groupId>org.apache.camel.quarkus</groupId>-->
<artifactId>camel-quarkus-telegram</artifactId> <!-- <artifactId>camel-quarkus-telegram</artifactId>-->
</dependency> <!-- </dependency>-->
<dependency> <dependency>
<groupId>org.apache.camel.quarkus</groupId> <groupId>org.apache.camel.quarkus</groupId>
<artifactId>camel-quarkus-core</artifactId> <artifactId>camel-quarkus-core</artifactId>
@ -49,20 +49,20 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.camel.quarkus</groupId> <groupId>org.apache.camel.quarkus</groupId>
<artifactId>camel-quarkus-sql</artifactId> <artifactId>camel-quarkus-couchdb</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-jdbc-postgresql</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.camel.quarkus</groupId> <groupId>org.apache.camel.quarkus</groupId>
<artifactId>camel-quarkus-direct</artifactId> <artifactId>camel-quarkus-jackson</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.camel.quarkus</groupId> <groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>camel-quarkus-log</artifactId> <artifactId>jackson-datatype-jsr310</artifactId>
</dependency> </dependency>
<!-- <dependency>-->
<!-- <groupId>io.quarkus</groupId>-->
<!-- <artifactId>quarkus-smallrye-reactive-messaging-rabbitmq</artifactId>-->
<!-- </dependency>-->
<dependency> <dependency>
<groupId>io.quarkus</groupId> <groupId>io.quarkus</groupId>
<artifactId>quarkus-arc</artifactId> <artifactId>quarkus-arc</artifactId>
@ -91,6 +91,11 @@
<artifactId>camel-quarkus-mock</artifactId> <artifactId>camel-quarkus-mock</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.apache.camel.quarkus</groupId>
<artifactId>camel-quarkus-direct</artifactId>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>

View file

@ -2,15 +2,12 @@ package com.c4181.beans;
import com.c4181.properties.AppProperties; import com.c4181.properties.AppProperties;
import com.google.maps.GeoApiContext; import com.google.maps.GeoApiContext;
import io.quarkus.runtime.annotations.RegisterForReflection;
import javax.enterprise.context.ApplicationScoped; import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.inject.Produces; import javax.enterprise.inject.Produces;
import javax.inject.Inject; import javax.inject.Inject;
@ApplicationScoped @ApplicationScoped
@RegisterForReflection(targets = {com.google.maps.GeocodingApi.Response.class,
com.google.maps.model.GeocodingResult.class}, registerFullHierarchy = true)
public class GoogleApiBeans { public class GoogleApiBeans {
@Inject @Inject

View file

@ -3,17 +3,15 @@ package com.c4181.camel;
import com.c4181.model.JsoCall; import com.c4181.model.JsoCall;
import com.c4181.model.JsoCallDecoder; import com.c4181.model.JsoCallDecoder;
import com.c4181.properties.AppProperties; import com.c4181.properties.AppProperties;
import io.quarkus.logging.Log; import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.camel.Exchange; import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.apache.camel.builder.RouteBuilder; import org.apache.camel.builder.RouteBuilder;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import javax.enterprise.context.ApplicationScoped; import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject; import javax.inject.Inject;
import java.awt.geom.Point2D;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
@ApplicationScoped @ApplicationScoped
public class CamelConfiguration extends RouteBuilder { public class CamelConfiguration extends RouteBuilder {
@ -21,58 +19,26 @@ public class CamelConfiguration extends RouteBuilder {
@Inject @Inject
AppProperties appProperties; AppProperties appProperties;
@Inject
JsoCallDecoder jsoCallDecoder;
@Override @Override
public void configure() { public void configure() {
ObjectMapper mapper = new ObjectMapper();
errorHandler(deadLetterChannel(appProperties.deadLetterRoute()) mapper.registerModule(new JavaTimeModule());
.onExceptionOccurred(exchange -> mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
Log.warnf("Failed to parse message in route %s. Sending to Dead Letter queue.", exchange.getProperty(Exchange.TO_ENDPOINT, String.class)))
.useOriginalMessage());
from(appProperties.jsoCadUpdateRouteIn()) from(appProperties.jsoCadUpdateRouteIn())
.filter(exchange -> StringUtils.isNotBlank(exchange.getIn().getBody(String.class))) .filter(exchange -> StringUtils.isNotBlank(exchange.getIn().getBody(String.class)))
.process((exchange -> { .process((exchange -> {
String updates = exchange.getIn().getBody(String.class); String updates = exchange.getIn().getBody(String.class);
List<JsoCall> jsoCalls = jsoCallDecoder.decodeJsoCallUpdates(updates); List<JsoCall> jsoCalls = JsoCallDecoder.decodeJsoCallUpdates(updates);
exchange.getIn().setBody(jsoCalls); exchange.getIn().setBody(jsoCalls);
})) }))
.removeHeader("*") .removeHeader("*")
.split(body()) .split(body())
.wireTap("direct:processedCalls")
.process(exchange -> { .process(exchange -> {
JsoCall jsoCall = exchange.getIn().getBody(JsoCall.class); String jsonString = mapper.writeValueAsString(exchange.getIn().getBody(JsoCall.class));
Map<String, Object> sqlCall = new HashMap<>(); exchange.getIn().setBody(jsonString);
sqlCall.put("incident_number", jsoCall.getIncidentNumber());
sqlCall.put("dispatched_time", jsoCall.getDispatchedTime());
sqlCall.put("address", jsoCall.getAddress());
sqlCall.put("signal", jsoCall.getSignal());
sqlCall.put("call_description", jsoCall.getCallDescription());
if (jsoCall.getPoint() != null) {
sqlCall.put("x", jsoCall.getPoint().getLat());
sqlCall.put("y", jsoCall.getPoint().getLng());
} else {
sqlCall.put("x", null);
sqlCall.put("y", null);
}
exchange.getIn().setBody(sqlCall);
}) })
.to(appProperties.jsoCadUpdateRouteOut()); .to(appProperties.jsoCadUpdateRouteOut());
from("direct:processedCalls")
.errorHandler(deadLetterChannel("log:dead?level=ERROR"))
.filter(exchange -> exchange.getIn().getBody(JsoCall.class).getPoint() != null)
.process(exchange -> {
JsoCall jsoCall = exchange.getIn().getBody(JsoCall.class);
if (Point2D.distance(jsoCall.getPoint().getLat(), jsoCall.getPoint().getLng(), appProperties.myLat(), appProperties.myLong())
<= appProperties.telegramNotificationThreshold()) {
exchange.setRouteStop(true);
}
})
.to(appProperties.telegramRoute());
} }
} }

View file

@ -10,7 +10,6 @@ public class JsoCall {
String address; String address;
String signal; String signal;
String callDescription; String callDescription;
Point point;
public String getIncidentNumber() { public String getIncidentNumber() {
return incidentNumber; return incidentNumber;
@ -52,25 +51,17 @@ public class JsoCall {
this.callDescription = callDescription; this.callDescription = callDescription;
} }
public Point getPoint() {
return point;
}
public void setPoint(Point point) {
this.point = point;
}
@Override @Override
public boolean equals(Object o) { public boolean equals(Object o) {
if (this == o) return true; if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false; if (o == null || getClass() != o.getClass()) return false;
JsoCall jsoCall = (JsoCall) o; JsoCall jsoCall = (JsoCall) o;
return Objects.equals(incidentNumber, jsoCall.incidentNumber) && Objects.equals(dispatchedTime, jsoCall.dispatchedTime) && Objects.equals(address, jsoCall.address) && Objects.equals(signal, jsoCall.signal) && Objects.equals(callDescription, jsoCall.callDescription) && Objects.equals(point, jsoCall.point); return Objects.equals(incidentNumber, jsoCall.incidentNumber) && Objects.equals(dispatchedTime, jsoCall.dispatchedTime) && Objects.equals(address, jsoCall.address) && Objects.equals(signal, jsoCall.signal) && Objects.equals(callDescription, jsoCall.callDescription);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(incidentNumber, dispatchedTime, address, signal, callDescription, point); return Objects.hash(incidentNumber, dispatchedTime, address, signal, callDescription);
} }
@Override @Override
@ -81,7 +72,6 @@ public class JsoCall {
", address='" + address + '\'' + ", address='" + address + '\'' +
", signal='" + signal + '\'' + ", signal='" + signal + '\'' +
", callDescription='" + callDescription + '\'' + ", callDescription='" + callDescription + '\'' +
", point=" + point +
'}'; '}';
} }
} }

View file

@ -6,7 +6,6 @@ import com.google.maps.GeocodingApiRequest;
import com.google.maps.model.GeocodingResult; import com.google.maps.model.GeocodingResult;
import io.quarkus.logging.Log; import io.quarkus.logging.Log;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject; import javax.inject.Inject;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneId; import java.time.ZoneId;
@ -17,44 +16,28 @@ import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ApplicationScoped
public class JsoCallDecoder { public class JsoCallDecoder {
@Inject @Inject
GeoApiContext geoApiContext; GeoApiContext geoApiContext;
public List<JsoCall> decodeJsoCallUpdates(String updates) { public static List<JsoCall> decodeJsoCallUpdates(String updates) {
List<String> newCalls = Arrays.stream(updates.split("\n")) List<String> newCalls = Arrays.stream(updates.split("\n"))
.filter(line -> line.contains("added")) .filter(line -> line.contains("added"))
.filter(line -> !line.contains("Last refreshed")) .filter(line -> !line.contains("Last refreshed"))
.map(line -> line.replace("(added ) ", "")) .map(line -> line.replace("(added ) ", ""))
.map(String::trim)
.toList(); .toList();
String pattern = "(\\d+)\\s+(\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2})\\s+(.+)\\s+( \\d+\\s?[\\w\\d]*)\\s+(.+)";
Pattern p = Pattern.compile(pattern);
List<JsoCall> jsoCalls = new ArrayList<>(); List<JsoCall> jsoCalls = new ArrayList<>();
for (String call : newCalls) { for (String call : newCalls) {
Matcher m = p.matcher(call); String trimmedCall = call.trim();
if (!m.matches() || m.groupCount() != 5) {
Log.warnf("Failed to parse call\n%s", call);
continue;
}
JsoCall jsoCall = new JsoCall(); JsoCall jsoCall = new JsoCall();
jsoCall.setIncidentNumber(m.group(1).trim()); jsoCall.setIncidentNumber(trimmedCall.substring(0, 12));
jsoCall.setDispatchedTime(parseTimeWithoutYear(m.group(2).trim())); jsoCall.setDispatchedTime(parseTimeWithoutYear(trimmedCall.substring(14, 25)));
jsoCall.setAddress(m.group(3).trim()); jsoCall.setAddress(trimmedCall.substring(27, 69).trim());
jsoCall.setSignal(m.group(4).trim()); jsoCall.setSignal(trimmedCall.substring(69, 77).trim());
jsoCall.setCallDescription(m.group(5).trim()); jsoCall.setCallDescription(trimmedCall.substring(77).trim());
if (!jsoCall.getAddress().contains("I95")
&& !jsoCall.getAddress().contains("I295") && !jsoCall.getAddress().contains("I10")) {
jsoCall.setPoint(geoCodeAddress(jsoCall.getAddress()));
}
jsoCalls.add(jsoCall); jsoCalls.add(jsoCall);
} }
@ -79,7 +62,7 @@ public class JsoCallDecoder {
private static LocalDateTime parseWithDefaultYear(String stringWithoutYear, int defaultYear) { private static LocalDateTime parseWithDefaultYear(String stringWithoutYear, int defaultYear) {
DateTimeFormatter parseFormatter = new DateTimeFormatterBuilder() DateTimeFormatter parseFormatter = new DateTimeFormatterBuilder()
.appendPattern("M-d HH:mm") .appendPattern("MM-dd HH:mm")
.parseDefaulting(ChronoField.YEAR, defaultYear) .parseDefaulting(ChronoField.YEAR, defaultYear)
.toFormatter(Locale.ENGLISH); .toFormatter(Locale.ENGLISH);
@ -88,7 +71,7 @@ public class JsoCallDecoder {
private Point geoCodeAddress(String address) { private Point geoCodeAddress(String address) {
GeocodingApiRequest request = GeocodingApi.newRequest(geoApiContext).address(address + "Jacksonville, FL"); GeocodingApiRequest request = GeocodingApi.newRequest(geoApiContext).address(address);
GeocodingResult[] results; GeocodingResult[] results;
try { try {
results = request.await(); results = request.await();
@ -97,7 +80,7 @@ public class JsoCallDecoder {
return null; return null;
} }
if (results == null || results[0] == null || results[0].geometry.location == null) { if (results == null || results[0] == null) {
return null; return null;
} }

View file

@ -7,10 +7,5 @@ public interface AppProperties {
String jsoCadUpdateRouteIn(); String jsoCadUpdateRouteIn();
String jsoCadUpdateRouteOut(); String jsoCadUpdateRouteOut();
String deadLetterRoute();
String telegramRoute();
String googleApiKey(); String googleApiKey();
double myLat();
double myLong();
double telegramNotificationThreshold();
} }

View file

@ -1,17 +1,3 @@
app.jso-cad-update-route-in=rabbitmq:${RABBITMQ_IP}/jso.cad.updates.to.postgres?queue=jso.cad.update.received&declare=false&vhost=jso&username=${RABBITMQ_USER}&password=${RABBITMQ_PASSWORD}&autoDelete=false app.jso-cad-update-route-in=rabbitmq:192.168.1.117/jso.cad.updates.to.couchdb?queue=jso.cad.update.received&declare=false&vhost=jso&username=${RABBITMQ_USER}&password=${RABBITMQ_PASSWORD}&autoDelete=false
app.jso-cad-update-route-out=sql:INSERT INTO calls(incident_number, dispatched_time, address, signal, call_description, point) VALUES (:#incident_number, :#dispatched_time, :#address, :#signal, :#call_description, point(:#x, :#y)) app.jso-cad-update-route-out=couchdb:http://192.168.1.220:5984/jso-calls?username=${COUCHDB_USER}&password=${COUCHDB_PASSWORD}
app.dead-letter-route=rabbitmq:${RABBITMQ_IP}/failed.updates?declare=false&vhost=jso&username=${RABBITMQ_USER}&password=${RABBITMQ_PASSWORD}&autoDelete=false
app.telegram-route=telegram:bots?authorizationToken=${TELEGRAM_BOT_ID}&chatId=${CHAT_ID}
app.google-api-key=${GOOGLE_API_KEY} app.google-api-key=${GOOGLE_API_KEY}
app.my-lat=30.3025061
app.my-long=-81.6436614
app.telegram-notification-threshold=2.0
RABBITMQ_IP=192.168.1.117
POSTGRES_IP=192.168.1.17
quarkus.datasource.db-kind=postgresql
quarkus.datasource.username=${POSTGRES_USER}
quarkus.datasource.password=${POSTGRES_PASSWORD}
quarkus.datasource.jdbc.url=jdbc:postgresql://${POSTGRES_IP}:5432/jsoCad

View file

@ -28,14 +28,14 @@ class CamelConfigTest {
@Test @Test
void happyPathTest() throws IOException { void happyPathTest() throws IOException {
MockEndpoint endpoint = (MockEndpoint) camelContext.getEndpoint(appProperties.jsoCadUpdateRouteOut()); // MockEndpoint endpoint = (MockEndpoint) camelContext.getEndpoint(appProperties.jsoCadUpdateRouteOut());
String data = Files.readString(Paths.get("src/test/resources/test-payload.txt")); String data = Files.readString(Paths.get("src/test/resources/test-payload.txt"));
producerTemplate.sendBody(appProperties.jsoCadUpdateRouteIn(), data); producerTemplate.sendBody(appProperties.jsoCadUpdateRouteIn(), data);
assertEquals(52, endpoint.getExchanges().size()); // assertEquals(52, endpoint.getExchanges().size());
JsoCall call = endpoint.getExchanges().get(0).getIn().getBody(JsoCall.class); // JsoCall call = endpoint.getExchanges().get(0).getIn().getBody(JsoCall.class);
assertEquals("202200769492", call.getIncidentNumber()); // assertEquals("202200769492", call.getIncidentNumber());
} }
} }

View file

@ -8,7 +8,6 @@ import java.nio.file.Paths;
import java.util.List; import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
class JsoCallDecoderTest { class JsoCallDecoderTest {
@ -16,59 +15,7 @@ class JsoCallDecoderTest {
void testDecode() throws IOException { void testDecode() throws IOException {
String data = Files.readString(Paths.get("src/test/resources/test-payload.txt")); String data = Files.readString(Paths.get("src/test/resources/test-payload.txt"));
JsoCallDecoder decoder = new JsoCallDecoder(); List<JsoCall> calls = JsoCallDecoder.decodeJsoCallUpdates(data);
List<JsoCall> calls = decoder.decodeJsoCallUpdates(data);
assertEquals(52, calls.size()); assertEquals(52, calls.size());
} }
@Test
void testEdgeCaseSignals() throws IOException {
String data = Files.readString(Paths.get("src/test/resources/edge-cases.txt"));
JsoCallDecoder decoder = new JsoCallDecoder();
List<JsoCall> calls = decoder.decodeJsoCallUpdates(data);
assertEquals(6, calls.size());
JsoCall firstCall = calls.get(0);
assertEquals("202200769492", firstCall.getIncidentNumber());
assertNotNull(firstCall.getDispatchedTime());
assertEquals("5200 RAMONA BLVD", firstCall.getAddress());
assertEquals("13", firstCall.getSignal());
assertEquals("SUSPICIOUS PERSON", firstCall.getCallDescription());
JsoCall secondCall = calls.get(1);
assertEquals("202200769474", secondCall.getIncidentNumber());
assertNotNull(secondCall.getDispatchedTime());
assertEquals("9100 MERRILL RD", secondCall.getAddress());
assertEquals("4", secondCall.getSignal());
assertEquals("AUTO CRASH", secondCall.getCallDescription());
JsoCall thirdCall = calls.get(2);
assertEquals("202200769409", thirdCall.getIncidentNumber());
assertNotNull(thirdCall.getDispatchedTime());
assertEquals("1000 ST CLAIR ST", thirdCall.getAddress());
assertEquals("21CT", thirdCall.getSignal());
assertEquals("BURGLARY CONVEYANCE TELESERVE", thirdCall.getCallDescription());
JsoCall fourthCall = calls.get(3);
assertEquals("202300004101", fourthCall.getIncidentNumber());
assertNotNull(fourthCall.getDispatchedTime());
assertEquals("12000 ATLANTIC BLVD", fourthCall.getAddress());
assertEquals("1050", fourthCall.getSignal());
assertEquals("TRAFFIC STOP", fourthCall.getCallDescription());
JsoCall fifthCall = calls.get(4);
assertEquals("202300004011", fifthCall.getIncidentNumber());
assertNotNull(fifthCall.getDispatchedTime());
assertEquals("BULLS BAY HWY / BEAVER ST W", fifthCall.getAddress());
assertEquals("0 13", fifthCall.getSignal());
assertEquals("ARMED SUSPICIOUS PERSON", fifthCall.getCallDescription());
JsoCall sixthCall = calls.get(5);
assertEquals("202300004560", sixthCall.getIncidentNumber());
assertNotNull(sixthCall.getDispatchedTime());
assertEquals("3700 TOLEDO RD", sixthCall.getAddress());
assertEquals("37", sixthCall.getSignal());
assertEquals("UNVERIFIED 911 CALL", sixthCall.getCallDescription());
}
} }

View file

@ -1,23 +0,0 @@
JSO CAD
JACKSONVILLE SHERIFF'S OFFICE
JSO Calls for Service
COMPLETED DISPATCHED CALLS FOR SERVICE
Welcome to the Jacksonville Sheriffs Office Completed Dispatched Calls for Service webpage. This page displays calls for service made to the Jacksonville Sheriff's Office that have recently been completed. The data on this page is refreshed automatically.
This information is not intended to be used as official crime data. This program does not provide information about all crimes, and excludes specific incidents such as sexual assaults and child abuse.
Disclaimer: The Jacksonville Sheriff's Office makes every effort to produce and publish current and accurate information. No warranties, expressed or implied, are provided for the data herein, its use, or its interpretation. The services provided are for informational purposes only and should not be relied on for any type of legal action.
(changed) Last refreshed 12/31 12:49:22
(into ) Last refreshed 12/31 13:45:54
Incident # Dispatched Block Address Signal Call Description
(added ) 202200769492 12-31 12:26 5200 RAMONA BLVD 13 SUSPICIOUS PERSON
(added ) 202200769474 12-31 12:19 9100 MERRILL RD 4 AUTO CRASH
(added ) 202200769409 12-31 11:18 1000 ST CLAIR ST 21CT BURGLARY CONVEYANCE TELESERVE
(added ) 202300004101 1-3 08:29 12000 ATLANTIC BLVD 1050 TRAFFIC STOP
(added ) 202300004011 1-3 07:37 BULLS BAY HWY / BEAVER ST W 0 13 ARMED SUSPICIOUS PERSON
(added ) 202300004560 1-3 11:44 3700 TOLEDO RD 37 UNVERIFIED 911 CALL