Skip to content

Commit

Permalink
Add LIC and PAL log support (#171)
Browse files Browse the repository at this point in the history 
Signed-off-by: Sanjula Ganepola <Sanjula.Ganepola@ibm.com>
  • Loading branch information
@SanjulaGanepola
SanjulaGanepola authored Dec 18, 2024
1 parent ffa181b commit 1f72320
Show file tree
Hide file tree
Showing 8 changed files with 208 additions and 78 deletions.
24 changes: 17 additions & 7 deletions camel/src/main/java/com/github/theprez/manzan/configuration/DataConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import org.ini4j.InvalidFileFormatException;

import com.github.theprez.jcmdutils.StringUtils;
import com.github.theprez.manzan.ManzanEventType;
import com.github.theprez.manzan.WatchStarter;
import com.github.theprez.manzan.routes.ManzanRoute;
import com.github.theprez.manzan.routes.event.FileEvent;
Expand Down Expand Up @@ -48,7 +49,7 @@ public synchronized Map<String, ManzanRoute> getRoutes() throws IOException, AS4
for (final String section : getIni().keySet()) {
final String type = getIni().get(section, "type");
if (StringUtils.isEmpty(type)) {
throw new RuntimeException("type not specified for data source [" + section + "]");
throw new RuntimeException("Type not specified for data source [" + section + "]");
}
if ("false".equalsIgnoreCase(getIni().get(section, "enabled"))) {
continue;
Expand All @@ -65,7 +66,7 @@ public synchronized Map<String, ManzanRoute> getRoutes() throws IOException, AS4
d = d.trim();
if (!m_destinations.contains(d)) {
throw new RuntimeException(
"no destination configured named '" + d + "' for data source '" + name + "'");
"No destination configured named '" + d + "' for data source '" + name + "'");
}
if (StringUtils.isNonEmpty(d)) {
destinations.add(d);
Expand All @@ -74,16 +75,25 @@ public synchronized Map<String, ManzanRoute> getRoutes() throws IOException, AS4
switch (type) {
case "watch":
String id = getRequiredString(name, "id");
String strwch = getRequiredString(name, "strwch");
String sqlRouteName = name + "sql";
String socketRouteName = name + "socket";

ret.put(sqlRouteName, new WatchMsgEventSql(sqlRouteName, id, format, destinations, schema, interval, numToProcess));
String strwch = getOptionalString(name, "strwch");
if (StringUtils.isNonEmpty(strwch)) {
WatchStarter ws = new WatchStarter(id, strwch);
ws.strwch();
ManzanEventType eventType;
if(strwch.contains("WCHMSGQ")) {
eventType = ManzanEventType.WATCH_MSG;
} else if(strwch.contains("WCHLICLOG")) {
eventType = ManzanEventType.WATCH_VLOG;
} else if(strwch.contains("WCHPAL")) {
eventType = ManzanEventType.WATCH_PAL;
} else {
throw new RuntimeException("Watch for message, LIC log entry, or PAL entry not specified");
}

ret.put(sqlRouteName, new WatchMsgEventSql(sqlRouteName, id, format, destinations, schema, eventType, interval, numToProcess));
ret.put(socketRouteName, new WatchMsgEventSockets(socketRouteName, format, destinations, schema, interval, numToProcess));
WatchStarter ws = new WatchStarter(id, strwch);
ws.strwch();
break;
case "file":
String file = getRequiredString(name, "file");
Expand Down
30 changes: 28 additions & 2 deletions camel/src/main/java/com/github/theprez/manzan/routes/ManzanRoute.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ public abstract class ManzanRoute extends RouteBuilder {
protected static final String EVENT_TYPE = "event_type";

protected static final String SESSION_ID = "SESSION_ID";
protected static final String MSG_ORDINAL_POSITION = "ORDINAL_POSITION";
protected static final String HANDLED_TIMESTAMP = "HANDLED_TIMESTAMP";

protected static final String MSG_MESSAGE_ID = "MESSAGE_ID";
protected static final String MSG_MESSAGE_TYPE = "MESSAGE_TYPE";
protected static final String MSG_SEVERITY = "SEVERITY";
Expand All @@ -22,10 +25,34 @@ public abstract class ManzanRoute extends RouteBuilder {
protected static final String MSG_SENDING_PROGRAM_NAME = "SENDING_PROGRAM_NAME";
protected static final String MSG_SENDING_MODULE_NAME = "SENDING_MODULE_NAME";
protected static final String MSG_SENDING_PROCEDURE_NAME = "SENDING_PROCEDURE_NAME";
protected static final String MSG_ORDINAL_POSITION = "ORDINAL_POSITION";
protected static final String MSG_MESSAGE_TIMESTAMP = "MESSAGE_TIMESTAMP";
protected static final String MSG_MESSAGE = "MESSAGE";

protected static final String MAJOR_CODE = "MAJOR_CODE";
protected static final String MINOR_CODE = "MINOR_CODE";
protected static final String LOG_ID = "LOG_ID";
protected static final String LOG_TIMESTAMP = "LOG_TIMESTAMP";
protected static final String TDE_NUM = "TDE_NUM";
protected static final String TASK_NAME = "TASK_NAME";
protected static final String SERVER_TYPE = "SERVER_TYPE";
protected static final String EXCEPTION_ID = "EXCEPTION_ID";
protected static final String THREAD_ID = "THREAD_ID";
protected static final String MODULE_OFFSET = "MODULE_OFFSET";
protected static final String MODULE_RU_NAME = "MODULE_RU_NAME";
protected static final String MODULE_NAME = "MODULE_NAME";
protected static final String MODULE_ENTRY_POINT_NAME = "MODULE_ENTRY_POINT_NAME";

protected static final String SYSTEM_REFERENCE_CODE = "SYSTEM_REFERENCE_CODE";
protected static final String DEVICE_NAME = "DEVICE_NAME";
protected static final String MODEL = "MODEL";
protected static final String SERIAL_NUMBER = "SERIAL_NUMBER";
protected static final String RESOURCE_NAME = "RESOURCE_NAME";
protected static final String PAL_TIMESTAMP = "PAL_TIMESTAMP";
protected static final String REFERENCE_CODE = "REFERENCE_CODE";
protected static final String SECONDARY_CODE = "SECONDARY_CODE";
protected static final String TABLE_ID = "TABLE_ID";
protected static final String SEQUENCE_NUM = "SEQUENCE_NUM";

protected static final int SEVERITY_LIMIT = 29;

protected static String getWatchName(final Exchange exchange) {
Expand Down Expand Up @@ -90,5 +117,4 @@ protected void setRecipientList(final List<String> _destinations) throws IOExcep
throw new IOException("Message watch for '" + m_name + "' has no valid destinations");
}
}

}
77 changes: 58 additions & 19 deletions camel/src/main/java/com/github/theprez/manzan/routes/dest/GrafanaLokiDestination.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,21 @@ public void run() {
public void configure() {
from(getInUri())
.routeId(m_name).process(exchange -> {
StreamBuilder builder = logController
.stream()
.l(appLabelName, appLabelValue)
.l(SESSION_ID, getWatchName(exchange));
String timestamp;
String[] keys;

final ManzanEventType type = (ManzanEventType) exchange.getIn().getHeader(EVENT_TYPE);
if (ManzanEventType.WATCH_MSG == type) {
StreamBuilder builder = logController
.stream()
.l(appLabelName, appLabelValue)
.l(Labels.LEVEL,
((Integer) get(exchange, MSG_SEVERITY)) > SEVERITY_LIMIT ? Labels.FATAL
: Labels.INFO)
.l(SESSION_ID, getWatchName(exchange));
if (type == ManzanEventType.WATCH_MSG) {
builder
.l(Labels.LEVEL, ((Integer) get(exchange, MSG_SEVERITY)) > SEVERITY_LIMIT ? Labels.FATAL
: Labels.INFO);

String[] keys = {
timestamp = MSG_MESSAGE_TIMESTAMP;
keys = new String[] {
MSG_MESSAGE_ID,
MSG_MESSAGE_TYPE,
MSG_SEVERITY,
Expand All @@ -64,20 +68,55 @@ public void configure() {
MSG_SENDING_MODULE_NAME,
MSG_SENDING_PROCEDURE_NAME
};
} else if (type == ManzanEventType.WATCH_VLOG) {
// TODO: Set log level

for (String key : keys) {
String value = getString(exchange, key);
if (!value.equals("")) {
builder.l(key, value);
}
}
timestamp = LOG_TIMESTAMP;
keys = new String[] {
MAJOR_CODE,
MINOR_CODE,
LOG_ID,
TDE_NUM,
TASK_NAME,
SERVER_TYPE,
EXCEPTION_ID,
JOB,
THREAD_ID,
MODULE_OFFSET,
MODULE_RU_NAME,
MODULE_NAME,
MODULE_ENTRY_POINT_NAME
};
} else if (type == ManzanEventType.WATCH_PAL) {
// TODO: Set log level

ILogStream stream = builder.build();
stream.log(Timestamp.valueOf(getString(exchange, MSG_MESSAGE_TIMESTAMP)).getTime(),
getBody(exchange, String.class));
timestamp = PAL_TIMESTAMP;
keys = new String[] {
SYSTEM_REFERENCE_CODE,
DEVICE_NAME,
MODEL,
SERIAL_NUMBER,
RESOURCE_NAME,
LOG_ID,
REFERENCE_CODE,
SECONDARY_CODE,
TABLE_ID,
SEQUENCE_NUM
};
} else {
throw new RuntimeException("Grafana Loki route doesn't know how to process type " + type);
}

for (String key : keys) {
String value = getString(exchange, key);
if (!value.equals("")) {
builder.l(key, value);
}
}

ILogStream stream = builder.build();
stream.log(Timestamp.valueOf(getString(exchange, timestamp)).getTime(),
getBody(exchange, String.class));
});
}
}
}
73 changes: 39 additions & 34 deletions camel/src/main/java/com/github/theprez/manzan/routes/dest/SentryDestination.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.github.theprez.manzan.routes.dest;

import java.sql.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
Expand Down Expand Up @@ -37,48 +38,52 @@ public void configure() {
.routeId(m_name)
.convertBodyTo(String.class)
.process(exchange -> {
final ManzanEventType type = (ManzanEventType) exchange.getIn().getHeader(EVENT_TYPE);
if (ManzanEventType.WATCH_MSG == type) {
System.out.println("sentry");
final SentryEvent event = new SentryEvent();
final String watch = getWatchName(exchange);
final SentryId id = new SentryId(UUID.randomUUID());
event.setTag("session id", watch);
event.setEventId(id);
event.setExtras(getDataMap(exchange));
final User user = new User();
user.setUsername(getString(exchange, MSG_SENDING_USRPRF));
event.setUser(user);
event.setPlatform("IBM i");
event.setTag("runtime", "IBM i");
event.setTag("runtime.name", "IBM i");
event.setDist("PASE");
event.setTransaction(getString(exchange, MSG_ORDINAL_POSITION));
final SentryEvent event = new SentryEvent();
event.setTag(SESSION_ID, getWatchName(exchange)); // TODO: Check if SESSION_ID or just session id
event.setEventId(new SentryId(UUID.randomUUID()));
event.setExtras(getDataMap(exchange));
event.setPlatform("IBM i");
event.setTag("runtime", "IBM i");
event.setTag("runtime.name", "IBM i");
event.setDist("PASE");
event.setTransaction(getString(exchange, MSG_ORDINAL_POSITION));

final User user = new User();
user.setUsername(getString(exchange, MSG_SENDING_USRPRF));
event.setUser(user);

final Message message = new Message();
message.setMessage(getBody(exchange, String.class));
event.setMessage(message);

SentryLevel level;
final int sev = (Integer) get(exchange, MSG_SEVERITY);
if (sev > SEVERITY_LIMIT) {
level = SentryLevel.ERROR;
} else {
level = SentryLevel.INFO;
}
String timestamp;
final List<String> fingerprints = new LinkedList<String>();

event.setLevel(level);
final Message message = new Message();
final String messageStr = getString(exchange, MSG_MESSAGE_ID) + ": "
+ getString(exchange, MSG_MESSAGE);
message.setMessage(messageStr);
final List<String> fingerprints = new LinkedList<String>();
final ManzanEventType type = (ManzanEventType) exchange.getIn().getHeader(EVENT_TYPE);
if (ManzanEventType.WATCH_MSG == type) {
event.setLevel(((Integer) get(exchange, MSG_SEVERITY)) > SEVERITY_LIMIT ? SentryLevel.ERROR
: SentryLevel.INFO);
timestamp = MSG_MESSAGE_TIMESTAMP;
fingerprints.add(getString(exchange, MSG_MESSAGE_ID));
fingerprints.add(getString(exchange, MSG_SENDING_PROCEDURE_NAME));
fingerprints.add(getString(exchange, MSG_SENDING_MODULE_NAME));
fingerprints.add(getString(exchange, MSG_SENDING_PROGRAM_NAME));
event.setFingerprints(fingerprints);
event.setMessage(message);
Sentry.captureEvent(event);
} else if (type == ManzanEventType.WATCH_VLOG) {
// TODO: Set log level
timestamp = LOG_TIMESTAMP;
fingerprints.add(getString(exchange, MAJOR_CODE));
fingerprints.add(getString(exchange, MINOR_CODE));
} else if (type == ManzanEventType.WATCH_PAL) {
// TODO: Set log level
timestamp = PAL_TIMESTAMP;
fingerprints.add(getString(exchange, SYSTEM_REFERENCE_CODE));
} else {
throw new RuntimeException("Sentry route doesn't know how to process type " + type);
}

event.setTimestamp(Date.valueOf(getString(exchange, timestamp))); // TODO: Verify date is valid
event.setFingerprints(fingerprints);
Sentry.captureEvent(event);
});
}
}
}
31 changes: 21 additions & 10 deletions camel/src/main/java/com/github/theprez/manzan/routes/event/WatchMsgEventSql.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,33 @@

public class WatchMsgEventSql extends ManzanRoute {

private final int m_interval;
private final int m_numToProcess;
private final String m_schema;
private final String m_sessionId;
private final ManzanEventType m_eventType;
private final String m_table;
private final int m_interval;
private final int m_numToProcess;
private final ManzanMessageFormatter m_formatter;

public WatchMsgEventSql(final String _name, final String _session_id, final String _format,
final List<String> _destinations, final String _schema, final int _interval, final int _numToProcess)
final List<String> _destinations, final String _schema, final ManzanEventType _eventType,
final int _interval, final int _numToProcess)
throws IOException {
super(_name);
m_schema = _schema;
m_sessionId = _session_id.trim().toUpperCase();
m_eventType = _eventType;
m_interval = _interval;
m_numToProcess = _numToProcess;
m_schema = _schema;
m_formatter = StringUtils.isEmpty(_format) ? null : new ManzanMessageFormatter(_format);
if (m_eventType == ManzanEventType.WATCH_MSG) {
m_table = "MANZANMSG";
} else if (m_eventType == ManzanEventType.WATCH_VLOG) {
m_table = "MANZANVLOG";
} else {
m_table = "MANZANPAL";
}
super.setRecipientList(_destinations);
m_sessionId = _session_id.trim().toUpperCase();
}

@Override
Expand All @@ -38,9 +49,9 @@ public void configure() {
// Reset the list of ordinal positions at the start of each execution
exchange.setProperty("ordinalPositions", new ArrayList<Integer>());
})
.setHeader(EVENT_TYPE, constant(ManzanEventType.WATCH_MSG))
.setBody(constant("SeLeCt * fRoM " + m_schema + ".mAnZaNmSg wHeRe SESSION_ID = '" + m_sessionId
+ "' limit " + m_numToProcess))
.setHeader(EVENT_TYPE, constant(m_eventType))
.setBody(constant("SELECT * FROM " + m_schema + "." + m_table + " WHERE SESSION_ID = '" + m_sessionId
+ "' LIMIT " + m_numToProcess))
.to("jdbc:jt400?outputType=StreamList")
.split(body()).streaming().parallelProcessing()
.setHeader("id", simple("${body[ORDINAL_POSITION]}"))
Expand Down Expand Up @@ -69,8 +80,8 @@ public void configure() {
.end()
.process(exchange -> {
// Constructing the WHERE clause for ORDINAL_POSITIONs
StringBuilder deleteQuery = new StringBuilder("DELETE FROM " + m_schema
+ ".MANZANMSG WHERE SESSION_ID = '" + m_sessionId + "' AND ORDINAL_POSITION IN (");
StringBuilder deleteQuery = new StringBuilder("DELETE FROM " + m_schema + "." + m_table
+ " WHERE SESSION_ID = '" + m_sessionId + "' AND ORDINAL_POSITION IN (");
@SuppressWarnings("unchecked")
List<Integer> ordinalPositions = exchange.getProperty("ordinalPositions", List.class);
if (ordinalPositions != null && !ordinalPositions.isEmpty()) {
Expand Down
2 changes: 1 addition & 1 deletion docs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ Many other destinations will be available. Examples include:
- [Mezmo](http://mezmo.com) ⏳
- [Microsoft Teams](http://teams.microsoft.com) ⏳
- [PagerDuty](http://pagerduty.com) ⏳
- [Sentry](http://sentry.io) 🌗
- [Sentry](http://sentry.io) ✅
- [Slack](http://slack.com) ✅
- SMS (via [Twilio](http://www.twilio.com)) ✅
- [Splunk](http://splunk.com) ⏳
Expand Down
Loading

0 comments on commit 1f72320

Please sign in to comment.