Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

🎨: building\java 中 Searcher 实现 Closeable 接口 #313

Closed
wants to merge 2 commits into from
Closed

🎨: building\java 中 Searcher 实现 Closeable 接口 #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fb8e0ad
🎨: building\java 中 Searcher 实现 Closeable 接口
MisfortuneCookie Aug 10, 2023
83f6993
🎨: 方法顺序调整
MisfortuneCookie Aug 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
55 changes: 21 additions & 34 deletions binding/java/ReadMe.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,35 +15,29 @@

```java
import org.lionsoul.ip2region.xdb.Searcher;

import java.io.*;
import java.util.concurrent.TimeUnit;

public class SearcherTest {
public static void main(String[] args) {
// 1、创建 searcher 对象
String dbPath = "ip2region.xdb file path";
Searcher searcher = null;
try {
searcher = Searcher.newWithFileOnly(dbPath);
} catch (IOException e) {
System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
return;
}
String ip = "1.2.3.4";

// 2、查询
try {
String ip = "1.2.3.4";
// 1、创建 searcher 对象
try (Searcher searcher = Searcher.newWithFileOnly(dbPath)) {
// 2、查询
long sTime = System.nanoTime();
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (IOException e) {
System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}
// 3、关闭资源（这里通过 try-with-resources 自动关闭）

// 3、关闭资源
searcher.close();

// 备注：并发使用，每个线程需要创建一个独立的 searcher 对象单独使用。
}
}
Expand All @@ -54,12 +48,14 @@ public class SearcherTest {
我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。
```java
import org.lionsoul.ip2region.xdb.Searcher;

import java.io.*;
import java.util.concurrent.TimeUnit;

public class SearcherTest {
public static void main(String[] args) {
String dbPath = "ip2region.xdb file path";
String ip = "1.2.3.4";

// 1、从 dbPath 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。
byte[] vIndex;
Expand All @@ -71,27 +67,17 @@ public class SearcherTest {
}

// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
Searcher searcher;
try {
searcher = Searcher.newWithVectorIndex(dbPath, vIndex);
} catch (Exception e) {
System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
return;
}

// 3、查询
try {
String ip = "1.2.3.4";
try (Searcher searcher = Searcher.newWithVectorIndex(dbPath, vIndex)) {
long sTime = System.nanoTime();
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (IOException e) {
System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}

// 4、关闭资源
searcher.close();
// 3、关闭资源（这里通过 try-with-resources 自动关闭）

// 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的制度 vIndex 缓存。
}
Expand All @@ -103,12 +89,14 @@ public class SearcherTest {
我们也可以预先加载整个 ip2region.xdb 的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。
```java
import org.lionsoul.ip2region.xdb.Searcher;

import java.io.*;
import java.util.concurrent.TimeUnit;

public class SearcherTest {
public static void main(String[] args) {
String dbPath = "ip2region.xdb file path";
String ip = "1.2.3.4";

// 1、从 dbPath 加载整个 xdb 到内存。
byte[] cBuff;
Expand All @@ -123,22 +111,21 @@ public class SearcherTest {
Searcher searcher;
try {
searcher = Searcher.newWithBuffer(cBuff);
} catch (Exception e) {
} catch (IOException e) {
System.out.printf("failed to create content cached searcher: %s\n", e);
return;
}

// 3、查询
try {
String ip = "1.2.3.4";
long sTime = System.nanoTime();
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}

// 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher
// searcher.close();

Expand Down
10 changes: 10 additions & 0 deletions binding/java/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<maven.compiler.source>1.6</maven.compiler.source>
<maven.compiler.target>1.6</maven.compiler.target>
</properties>
Expand Down Expand Up @@ -112,6 +113,15 @@
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
</plugins>
</build>

Expand Down
157 changes: 80 additions & 77 deletions binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@

import org.lionsoul.ip2region.xdb.Searcher;

import java.io.*;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.concurrent.TimeUnit;

public class SearchTest {
Expand Down Expand Up @@ -65,40 +68,38 @@ public static void searchTest(String[] args) throws IOException {
}
}

if (dbPath.length() < 1) {
if (dbPath.isEmpty()) {
System.out.print("java -jar ip2region-{version}.jar search [command options]\n");
System.out.print("options:\n");
System.out.print(" --db string ip2region binary xdb file path\n");
System.out.print(" --cache-policy string cache policy: file/vectorIndex/content\n");
return;
}

Searcher searcher = createSearcher(dbPath, cachePolicy);
final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy);
while ( true ) {
System.out.print("ip2region>> ");
String line = reader.readLine().trim();
if ( line.length() < 2 ) {
continue;
}
try (Searcher searcher = createSearcher(dbPath, cachePolicy);
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy);
while (true) {
System.out.print("ip2region>> ");
String line = reader.readLine().trim();
if (line.length() < 2) {
continue;
}

if ( line.equalsIgnoreCase("quit") ) {
break;
}
if ("quit".equalsIgnoreCase(line)) {
break;
}

try {
double sTime = System.nanoTime();
String region = searcher.search(line);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount());
try {
double sTime = System.nanoTime();
String region = searcher.search(line);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount());
}
}
}

reader.close();
searcher.close();
System.out.println("searcher test program exited, thanks for trying");
}

Expand All @@ -121,19 +122,23 @@ public static void benchTest(String[] args) throws IOException {

String key = r.substring(2, sIdx);
String val = r.substring(sIdx + 1);
if ("db".equals(key)) {
dbPath = val;
} else if ("src".equals(key)) {
srcPath = val;
} else if ("cache-policy".equals(key)) {
cachePolicy = val;
} else {
System.out.printf("undefined option `%s`\n", r);
return;
switch (key) {
case "db":
dbPath = val;
break;
case "src":
srcPath = val;
break;
case "cache-policy":
cachePolicy = val;
break;
default:
System.out.printf("undefined option `%s`\n", r);
return;
}
}

if (dbPath.length() < 1 || srcPath.length() < 1) {
if (dbPath.isEmpty() || srcPath.isEmpty()) {
System.out.print("java -jar ip2region-{version}.jar bench [command options]\n");
System.out.print("options:\n");
System.out.print(" --db string ip2region binary xdb file path\n");
Expand All @@ -142,61 +147,59 @@ public static void benchTest(String[] args) throws IOException {
return;
}

Searcher searcher = createSearcher(dbPath, cachePolicy);
long count = 0, costs = 0, tStart = System.nanoTime();
String line;
final BufferedReader reader = new BufferedReader(new FileReader(srcPath));
while ((line = reader.readLine()) != null) {
String l = line.trim();
String[] ps = l.split("\\|", 3);
if (ps.length != 3) {
System.out.printf("invalid ip segment `%s`\n", l);
return;
}

long sip;
try {
sip = Searcher.checkIP(ps[0]);
} catch (Exception e) {
System.out.printf("check start ip `%s`: %s\n", ps[0], e);
return;
}

long eip;
try {
eip = Searcher.checkIP(ps[1]);
} catch (Exception e) {
System.out.printf("check end ip `%s`: %s\n", ps[1], e);
return;
}
try (Searcher searcher = createSearcher(dbPath, cachePolicy);
BufferedReader reader = new BufferedReader(new FileReader(srcPath))) {
String line;
while ((line = reader.readLine()) != null) {
String l = line.trim();
String[] ps = l.split("\\|", 3);
if (ps.length != 3) {
System.out.printf("invalid ip segment `%s`\n", l);
return;
}

if (sip > eip) {
System.out.printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]);
return;
}
long sip;
try {
sip = Searcher.checkIP(ps[0]);
} catch (Exception e) {
System.out.printf("check start ip `%s`: %s\n", ps[0], e);
return;
}

long mip = (sip + eip) >> 1;
for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) {
long sTime = System.nanoTime();
String region = searcher.search(ip);
costs += System.nanoTime() - sTime;
long eip;
try {
eip = Searcher.checkIP(ps[1]);
} catch (Exception e) {
System.out.printf("check end ip `%s`: %s\n", ps[1], e);
return;
}

// check the region info
if (!ps[2].equals(region)) {
System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]);
if (sip > eip) {
System.out.printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]);
return;
}

count++;
long mip = (sip + eip) >> 1;
for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) {
long sTime = System.nanoTime();
String region = searcher.search(ip);
costs += System.nanoTime() - sTime;

// check the region info
if (!ps[2].equals(region)) {
System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]);
return;
}

count++;
}
}
}

reader.close();
searcher.close();
long took = System.nanoTime() - tStart;
System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n",
cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took),
count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs/count));
count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs / count));
}

public static void main(String[] args) {
Expand Down
2 changes: 1 addition & 1 deletion binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public class UtilTest {

public static void testIP2Long() {
String ip = "1.2.3.4";
long ipAddr = 0;
long ipAddr;
try {
ipAddr = Searcher.checkIP(ip);
} catch (Exception e) {
Expand Down
15 changes: 8 additions & 7 deletions binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@ public Header(byte[] buff) {
buffer = buff;
}

@Override public String toString() {
@Override
public String toString() {
return "{" +
"Version: " + version + ',' +
"IndexPolicy: " + indexPolicy + ',' +
"CreatedAt: " + createdAt + ',' +
"StartIndexPtr: " + startIndexPtr + ',' +
"EndIndexPtr: " + endIndexPtr +
'}';
"Version: " + version + ',' +
"IndexPolicy: " + indexPolicy + ',' +
"CreatedAt: " + createdAt + ',' +
"StartIndexPtr: " + startIndexPtr + ',' +
"EndIndexPtr: " + endIndexPtr +
'}';
}
}