Java源码示例：nl.basjes.parse.useragent.UserAgent
示例1
private static void printAgent(OutputFormat outputFormat, List<String> fields, UserAgent agent) {
switch (outputFormat) {
case CSV:
boolean doSeparator = false;
for (String field : fields) {
if (doSeparator) {
System.out.print("\t");
} else {
doSeparator = true;
}
String value = agent.getValue(field);
if (value != null) {
System.out.print(value);
}
}
System.out.println();
break;
case JSON:
System.out.println(agent.toJson(fields));
break;
case YAML:
System.out.println(agent.toYamlTestCase());
break;
default:
}
}
示例2
private void addBugReportButton(StringBuilder sb, UserAgent userAgent) {
// https://github.com/nielsbasjes/yauaa/issues/new?title=Bug%20report&body=bar
try {
StringBuilder reportUrl = new StringBuilder("https://github.com/nielsbasjes/yauaa/issues/new?title=Bug%20report&body=");
String report = "I found a problem with this useragent.\n" +
"[Please update the output below to match what you expect it should be]\n" +
"\n```\n" +
userAgent.toYamlTestCase().replaceAll(" +:", " :") +
"\n```\n";
reportUrl.append(URLEncoder.encode(report, "UTF-8"));
String githubUrl = "https://github.com/login?return_to=" + URLEncoder.encode(reportUrl.toString(), "UTF-8");
sb.append("If you find a problem with this result then please report a bug here: " +
"<a href=\"").append(githubUrl).append("\">Yauaa issue report</a>");
} catch (UnsupportedEncodingException e) {
// Never happens.
}
}
示例3
public T map(T record) {
if (record == null) {
return null;
}
if (mapper == null) {
throw new InvalidParserConfigurationException("[Map] The mapper instance is null.");
}
UserAgent userAgent = userAgentAnalyzer.parse(mapper.getUserAgentString(record));
for (Map.Entry<String, List<Method>> fieldSetter : fieldSetters.entrySet()) {
String value = userAgent.getValue(fieldSetter.getKey());
for (Method method : fieldSetter.getValue()) {
try {
method.invoke(mapper, record, value);
} catch (IllegalAccessException | InvocationTargetException e) {
throw new InvalidParserConfigurationException("A problem occurred while calling the requested setter", e);
}
}
}
return record;
}
示例4
public static DeviceClass getDeviceClass(UserAgent userAgent) {
switch (userAgent.getValue(DEVICE_CLASS)) {
case "Desktop": return DESKTOP;
case "Anonymized": return ANONYMIZED;
case "Mobile": return MOBILE;
case "Tablet": return TABLET;
case "Phone": return PHONE;
case "Watch": return WATCH;
case "Virtual Reality": return VIRTUAL_REALITY;
case "eReader": return E_READER;
case "Set-top box": return SET_TOP_BOX;
case "TV": return TV;
case "Game Console": return GAME_CONSOLE;
case "Handheld Game Console": return HANDHELD_GAME_CONSOLE;
case "Robot": return ROBOT;
case "Robot Mobile": return ROBOT_MOBILE;
case "Robot Imitator": return ROBOT_IMITATOR;
case "Hacker": return HACKER;
case "Unknown": return UNKNOWN;
default: return UNCLASSIFIED;
}
}
示例5
@Test
public void testLoadAdditionalRules() {
UserAgentAnalyzer userAgentAnalyzer =
UserAgentAnalyzer
.newBuilder()
.withField("DeviceClass")
.withoutCache()
.hideMatcherLoadStats()
.addResources("ExtraLoadedRule1.yaml")
.withField("ExtraValue2")
.withField("ExtraValue1")
.addResources("ExtraLoadedRule2.yaml")
.build();
UserAgent parsedAgent = userAgentAnalyzer.parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");
// The requested fields
assertEquals("Phone", parsedAgent.getValue("DeviceClass" ));
assertEquals("One", parsedAgent.getValue("ExtraValue1" ));
assertEquals("Two", parsedAgent.getValue("ExtraValue2" ));
}
示例6
@Test
public void testLoadOnlyCustomRules() {
UserAgentAnalyzer userAgentAnalyzer =
UserAgentAnalyzer
.newBuilder()
.withoutCache()
.hideMatcherLoadStats()
.addResources("ExtraLoadedRule1.yaml")
.withField("ExtraValue2")
.withField("ExtraValue1")
.addResources("ExtraLoadedRule2.yaml")
.build();
UserAgent parsedAgent = userAgentAnalyzer.parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");
// The requested fields
assertEquals("One", parsedAgent.getValue("ExtraValue1" ));
assertEquals("Two", parsedAgent.getValue("ExtraValue2" ));
}
示例7
@Test
public void testLoadOnlyCompanyCustomFormatRules() {
UserAgentAnalyzer userAgentAnalyzer =
UserAgentAnalyzer
.newBuilder()
.withoutCache()
.hideMatcherLoadStats()
.dropDefaultResources()
.addResources("CompanyInternalUserAgents.yaml")
.withFields("ApplicationName", "ApplicationVersion")
.withFields(Arrays.asList("ApplicationInstance", "ApplicationGitCommit"))
.withField("ServerName")
.build();
UserAgent parsedAgent = userAgentAnalyzer.parse(
"TestApplication/1.2.3 (node123.datacenter.example.nl; 1234; d71922715c2bfe29343644b14a4731bf5690e66e)");
// The requested fields
assertEquals("TestApplication", parsedAgent.getValue("ApplicationName"));
assertEquals("1.2.3", parsedAgent.getValue("ApplicationVersion"));
assertEquals("1234", parsedAgent.getValue("ApplicationInstance"));
assertEquals("d71922715c2bfe29343644b14a4731bf5690e66e", parsedAgent.getValue("ApplicationGitCommit"));
assertEquals("node123.datacenter.example.nl", parsedAgent.getValue("ServerName"));
}
示例8
@Override
public Object evaluate(DeferredObject[] args) throws HiveException {
String userAgentString = useragentOI.getPrimitiveJavaObject(args[0].get());
if (userAgentString == null) {
return null;
}
UserAgent userAgent = userAgentAnalyzer.parse(userAgentString);
List<Object> result = new ArrayList<>(fieldNames.size());
for (String fieldName : fieldNames) {
String value = userAgent.getValue(fieldName);
if (value == null) {
result.add(null);
} else {
result.add(new Text(value));
}
}
return result.toArray();
}
示例9
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { // NOSONAR: Explicitly name the exception
FlowFile flowFile = session.get();
String userAgentString = flowFile.getAttribute(USERAGENTSTRING_ATTRIBUTENAME);
if (userAgentString == null) {
session.transfer(flowFile, MISSING);
} else {
UserAgent userAgent = uaa.parse(userAgentString);
for (String fieldName : extractFieldNames) {
String fieldValue = userAgent.getValue(fieldName);
flowFile = session.putAttribute(flowFile, ATTRIBUTE_PREFIX + fieldName, fieldValue);
}
session.transfer(flowFile, SUCCESS);
}
session.commit();
}
示例10
@Override
public Collection<Event> filter(Collection<Event> events, FilterMatchListener filterMatchListener) {
for (Event event : events) {
Object rawField = event.getField(sourceField);
if (rawField instanceof String) {
String userAgentString = (String)rawField;
UserAgent agent = userAgentAnalyzer.parse(userAgentString);
for (String fieldName : requestedFieldNames) {
event.setField(outputFields.get(fieldName), agent.getValue(fieldName));
}
}
}
return events;
}
示例11
@Override
public void dissect(Parsable<?> parsable, String inputname) throws DissectionFailure {
final ParsedField agentField = parsable.getParsableField(INPUT_TYPE, inputname);
String userAgentString = agentField.getValue().getString();
if (userAgentString == null) {
return; // Weird, but it happens
}
UserAgent agent = userAgentAnalyzer.parse(userAgentString);
for (String fieldName : requestedFieldNames) {
parsable.addDissection(inputname, getFieldOutputType(fieldName), fieldNameToDissectionName(fieldName), agent.getValue(fieldName));
}
}
示例12
@Test
public void testParser() {
Demo demo = new Demo();
String userAgent = "Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36";
UserAgent result = demo.parse(userAgent);
assertTrue(result.toXML().contains("<DeviceName>Google Nexus 6</DeviceName>"),
"The parser must extract the correct DeviceName");
}
示例13
boolean analyzeMatchersResult() {
boolean passed = true;
for (String fieldName : getAvailableFieldNamesSorted()) {
Map<Long, String> receivedValues = new HashMap<>(32);
for (Pair<UserAgent, Matcher> pair: appliedMatcherResults) {
UserAgent result = pair.getLeft();
AgentField partialField = result.get(fieldName);
if (partialField != null && partialField.getConfidence() >= 0) {
String previousValue = receivedValues.get(partialField.getConfidence());
if (previousValue != null) {
if (!previousValue.equals(partialField.getValue())) {
if (passed) {
LOG.error("***********************************************************");
LOG.error("*** REALLY IMPORTANT ERRORS IN THE RULESET ***");
LOG.error("*** YOU MUST CHANGE THE CONFIDENCE LEVELS OF YOUR RULES ***");
LOG.error("***********************************************************");
}
passed = false;
LOG.error("Found different value for \"{}\" with SAME confidence {}: \"{}\" and \"{}\"",
fieldName, partialField.getConfidence(), previousValue, partialField.getValue());
}
} else {
receivedValues.put(partialField.getConfidence(), partialField.getValue());
}
}
}
}
return passed;
}
示例14
/**
* @param userAgent The instance that needs to be classified.
* @return Is this a 'normal' consumer device that can simply be bought/downloaded and used as intended.
*/
public static boolean isNormalConsumerDevice(UserAgent userAgent) {
switch (getDeviceClass(userAgent)) {
case DESKTOP:
case MOBILE:
case TABLET:
case PHONE:
case WATCH:
case VIRTUAL_REALITY:
case E_READER:
case SET_TOP_BOX:
case TV:
case GAME_CONSOLE:
case HANDHELD_GAME_CONSOLE:
return true;
case ANONYMIZED:
case ROBOT:
case ROBOT_MOBILE:
case ROBOT_IMITATOR:
case HACKER:
case UNKNOWN:
case UNCLASSIFIED:
default:
return false;
}
}
示例15
/**
* @param userAgent The instance that needs to be classified.
* @return Is this a 'mobile' device. (includes robots that want to be treated as mobile)
*/
public static boolean isMobile(UserAgent userAgent) {
switch (getDeviceClass(userAgent)) {
case MOBILE:
case TABLET:
case PHONE:
case WATCH:
case VIRTUAL_REALITY:
case E_READER:
case HANDHELD_GAME_CONSOLE:
case ROBOT_MOBILE:
return true;
case DESKTOP:
case SET_TOP_BOX:
case TV:
case GAME_CONSOLE:
case ANONYMIZED:
case ROBOT:
case ROBOT_IMITATOR:
case HACKER:
case UNKNOWN:
case UNCLASSIFIED:
default:
return false;
}
}
示例16
/**
* @param userAgent The instance that needs to be classified.
* @return If this is probably a human using the device.
*/
public static boolean isHuman(UserAgent userAgent) {
switch (getDeviceClass(userAgent)) {
case DESKTOP:
case MOBILE:
case TABLET:
case PHONE:
case WATCH:
case VIRTUAL_REALITY:
case E_READER:
case SET_TOP_BOX:
case TV:
case GAME_CONSOLE:
case HANDHELD_GAME_CONSOLE:
case ANONYMIZED:
return true;
case ROBOT:
case ROBOT_MOBILE:
case ROBOT_IMITATOR:
case HACKER:
case UNKNOWN:
case UNCLASSIFIED:
default:
return false;
}
}
示例17
/**
* @param userAgent The instance that needs to be classified.
* @return Do we see this as deliberate misuse?
*/
public static boolean isDeliberateMisuse(UserAgent userAgent) {
switch (getDeviceClass(userAgent)) {
case ANONYMIZED:
case ROBOT_IMITATOR:
case HACKER:
return true;
case DESKTOP:
case MOBILE:
case TABLET:
case PHONE:
case WATCH:
case VIRTUAL_REALITY:
case E_READER:
case SET_TOP_BOX:
case TV:
case GAME_CONSOLE:
case HANDHELD_GAME_CONSOLE:
case ROBOT:
case ROBOT_MOBILE:
case UNKNOWN:
case UNCLASSIFIED:
default:
return false;
}
}
示例18
private void runTestCase(AbstractUserAgentAnalyzerDirect userAgentAnalyzer) {
UserAgent parsedAgent = userAgentAnalyzer.parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");
LoggerFactory.getLogger(TestBuilder.class).info("{}", parsedAgent.toYamlTestCase(true));
// The requested fields
assertEquals("Phone", parsedAgent.getValue("DeviceClass" )); // Phone
assertEquals("Chrome 53", parsedAgent.getValue("AgentNameVersionMajor" )); // Chrome 53
// The fields that are internally needed to build the requested fields
assertEquals("Chrome", parsedAgent.getValue("AgentName" )); // Chrome
assertEquals("53.0.2785.124", parsedAgent.getValue("AgentVersion" )); // 53.0.2785.124
assertEquals("53", parsedAgent.getValue("AgentVersionMajor" )); // 53
// The rest must be the default value (i.e. no rules fired)
// FIXME:
// assertTrue(parsedAgent.get("DeviceName" ).isDefaultValue()); // Nexus 6
// assertTrue(parsedAgent.get("DeviceBrand" ).isDefaultValue()); // Google
// assertTrue(parsedAgent.get("OperatingSystemClass" ).isDefaultValue()); // Mobile
// assertTrue(parsedAgent.get("OperatingSystemName" ).isDefaultValue()); // Android
// assertTrue(parsedAgent.get("OperatingSystemVersion" ).isDefaultValue()); // 7.0
// assertTrue(parsedAgent.get("OperatingSystemNameVersion" ).isDefaultValue()); // Android 7.0
// assertTrue(parsedAgent.get("OperatingSystemVersionBuild" ).isDefaultValue()); // NBD90Z
// assertTrue(parsedAgent.get("LayoutEngineClass" ).isDefaultValue()); // Browser
// assertTrue(parsedAgent.get("LayoutEngineName" ).isDefaultValue()); // Blink
// assertTrue(parsedAgent.get("LayoutEngineVersion" ).isDefaultValue()); // 53.0
// assertTrue(parsedAgent.get("LayoutEngineVersionMajor" ).isDefaultValue()); // 53
// assertTrue(parsedAgent.get("LayoutEngineNameVersion" ).isDefaultValue()); // Blink 53.0
// assertTrue(parsedAgent.get("LayoutEngineNameVersionMajor" ).isDefaultValue()); // Blink 53
// assertTrue(parsedAgent.get("AgentClass" ).isDefaultValue()); // Browser
// assertTrue(parsedAgent.get("AgentNameVersion" ).isDefaultValue()); // Chrome 53.0.2785.124
}
示例19
private void doTest(Triple<Counter, String, String> test) {
long start = System.nanoTime();
UserAgent agent = uaa.parse(test.getRight());
long stop = System.nanoTime();
if(agent.getValue("DeviceClass") == null) {
LOG.error("This should not happen, yet we keep this test to avoid the parse being optimized out.");
}
test.getLeft().increment((double)(stop - start));
}
示例20
@Override
public Tuple exec(Tuple tuple) throws IOException {
initialize();
String userAgentString = (String) tuple.get(0);
UserAgent agent = analyzer.parse(userAgentString);
Tuple result = TUPLE_FACTORY.newTuple();
for (String fieldName: requestedFields) {
result.append(agent.getValue(fieldName));
}
return result;
}
示例21
@Override
public IngestDocument execute(IngestDocument ingestDocument) {
String content = ingestDocument.getFieldValue(field, String.class);
UserAgent userAgent = uaa.parse(content);
Map<String, String> resultMap = userAgent.toMap();
resultMap.remove(USERAGENT_FIELDNAME);
ingestDocument.setFieldValue(targetField, resultMap);
return ingestDocument;
}
示例22
public UserAgent parse(String userAgent) {
return uaa.parse(userAgent);
}
示例23
String toMatchTrace(List<String> highlightNames) {
StringBuilder sb = new StringBuilder(4096);
sb.append('\n');
sb.append("+=========================================+\n");
sb.append("| Matcher results that have been combined |\n");
sb.append("+=========================================+\n");
sb.append('\n');
appliedMatcherResults.sort((o1, o2) -> {
Matcher m1 = o1.getValue();
Matcher m2 = o2.getValue();
return m1.getMatcherSourceLocation().compareTo(m2.getMatcherSourceLocation());
});
for (Pair<UserAgent, Matcher> pair: appliedMatcherResults){
sb.append('\n');
sb.append("+================\n");
sb.append("+ Applied matcher\n");
sb.append("+----------------\n");
UserAgent result = pair.getLeft();
Matcher matcher = pair.getRight();
sb.append(matcher.toString());
sb.append("+----------------\n");
sb.append("+ Results\n");
sb.append("+----------------\n");
for (String fieldName : result.getAvailableFieldNamesSorted()) {
AgentField field = result.get(fieldName);
// if (field == null) {
// LOG.error("Should not happen: No such field: {}", fieldName);
// } else {
if (field.getConfidence() >= 0) {
String marker = "";
if (highlightNames.contains(fieldName)) {
marker = " <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<";
}
sb.append("| ").append(fieldName).append('(').append(field.getConfidence());
if (field.isDefaultValue()) {
sb.append(" => isDefaultValue");
}
sb.append(") = ").append(field.getValue()).append(marker).append('\n');
}
// }
}
sb.append("+================\n");
}
return sb.toString();
}
示例24
public UserAgent parse(String userAgentString) {
MutableUserAgent userAgent = new MutableUserAgent(userAgentString);
return parseIntoCleanUserAgent(userAgent);
}
示例25
@Benchmark
public UserAgent android6Chrome46(ThreadState state) {
return state.uaa.parse("Mozilla/5.0 (Linux; Android 6.0; Nexus 6 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) " +
"Chrome/46.0.2490.76 Mobile Safari/537.36");
}
示例26
@Benchmark
public UserAgent androidPhone(ThreadState state) {
return state.uaa.parse("Mozilla/5.0 (Linux; Android 5.0.1; ALE-L21 Build/HuaweiALE-L21) AppleWebKit/537.36 (KHTML, like Gecko) " +
"Version/4.0 Chrome/37.0.0.0 Mobile Safari/537.36");
}
示例27
@Benchmark
public UserAgent googlebot(ThreadState state) {
return state.uaa.parse("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
}
示例28
@Benchmark
public UserAgent googleBotMobileAndroid(ThreadState state) {
return state.uaa.parse("Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) " +
"Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
}
示例29
@Benchmark
public UserAgent googleAdsBot(ThreadState state) {
return state.uaa.parse("AdsBot-Google (+http://www.google.com/adsbot.html)");
}
示例30
@Benchmark
public UserAgent googleAdsBotMobile(ThreadState state) {
return state.uaa.parse("Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) " +
"Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)");
}
