测试代码
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
package com.boot.demo.test.io;
import java.io.*;
import java.lang.reflect.Method;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.security.AccessController;
import java.security.PrivilegedAction;
/**
* @author braska
* @date 2020/3/19
**/
public class FileTest {
public static void fileStream(String sourceFile, String targetFile) {
File file = new File(targetFile);
try (FileInputStream fis = new FileInputStream(sourceFile);
FileOutputStream fos = new FileOutputStream(file)) {
byte[] bytes = new byte[1024 * 1024];
int len;
while ((len = fis.read(bytes)) > 0) {
fos.write(bytes, 0, len);
}
} catch (Exception e) {
}
}
public static void bufferStream(String sourceFile, String targetFile) {
try (BufferedInputStream bis = new BufferedInputStream(Files.newInputStream(Paths.get(sourceFile)));
BufferedOutputStream bos =
new BufferedOutputStream(Files.newOutputStream(Paths.get(targetFile),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE))) {
byte[] bytes = new byte[1024 * 1024];
int len;
while ((len = bis.read(bytes)) > 0) {
bos.write(bytes, 0, len);
}
} catch (Exception e) {
e.printStackTrace();
}
}
public static void randomFile(String sourceFile, String targetFile) {
try (RandomAccessFile read = new RandomAccessFile(sourceFile, "r");
RandomAccessFile write = new RandomAccessFile(targetFile, "rw")) {
byte[] bytes = new byte[1024 * 1024];
int len;
while ((len = read.read(bytes)) > 0) {
write.write(bytes, 0, len);
}
} catch (Exception e) {
}
}
public static void memoryMap(String sourceFile, String targetFile) {
try (FileChannel rc = FileChannel.open(Paths.get(sourceFile));
FileChannel wc = FileChannel.open(Paths.get(targetFile),
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE)) {
long copy = 1L << 30;
long cur = 0;
long fileLength = rc.size();
while (cur < fileLength) {
copy = cur + copy > fileLength ? (fileLength - cur) : copy;
MappedByteBuffer rMap = rc.map(FileChannel.MapMode.READ_ONLY, cur, copy);
MappedByteBuffer wMap = wc.map(FileChannel.MapMode.READ_WRITE, cur, copy);
for (int i = 0; i < copy; i++) {
byte b = rMap.get(i); //从源文件读取字节
wMap.put(i, b); //把字节写到目标文件中
}
System.gc(); //手动调用 GC <必须的，否则出现异常>
System.runFinalization();
cur += copy;
}
} catch (Exception e) {
e.printStackTrace();
}
}
private static String buildFilePath(String path, String fileName, String extension) {
return String.format("%s%s.%s", path, fileName, extension);
}
public static void main(String[] args) {
/* String path = "F:\\workspace\\demo\\";
String extension = "hprof";
// 30M文件
String sourceFile = buildFilePath(path, "01", extension);*/
/* String path = "E:\\software\\";
String extension = "exe";
// 460M文件
String sourceFile = buildFilePath(path, "Anaconda3-2019.10-Windows-x86_64", extension);*/
String path = "E:\\software\\";
String extension = "zip";
// 1.47G文件
String sourceFile = buildFilePath(path, "software", extension);
String targetFile;
long start;
/* targetFile = buildFilePath(path, "target_file_stream", extension);
start = System.currentTimeMillis();
FileTest.fileStream(sourceFile, targetFile);
System.out.println("file stream used time:" + (System.currentTimeMillis() - start));*/
/* targetFile = buildFilePath(path, "target_buffer_stream", extension);
start = System.currentTimeMillis();
FileTest.bufferStream(sourceFile, targetFile);
System.out.println("buffer stream used time:" + (System.currentTimeMillis() - start));*/
/* targetFile = buildFilePath(path, "target_random_file", extension);
start = System.currentTimeMillis();
FileTest.randomFile(sourceFile, targetFile);
System.out.println("random file used time:" + (System.currentTimeMillis() - start));*/
targetFile = buildFilePath(path, "target_memory_map", extension);
start = System.currentTimeMillis();
FileTest.memoryMap(sourceFile, targetFile);
System.out.println("memory map used time:" + (System.currentTimeMillis() - start));
}
}
测试结果
文件大小读写方式耗时
30M
普通文件流
50-60 ms
缓存流
32-35 ms
随机文件方式
40-50 ms
内存映射文件
50-60 ms
461M
普通文件流
1300-2300 ms
缓存流
1700-2000 ms
随机文件方式
1300-3000 ms
内存映射文件
890-1000 ms
1.47G
普通文件流
11s
缓存流
9s
随机文件方式
10s
内存映射文件
3s(首次较慢)
结尾：测试1.47G大文件时，内存映射文件中copy大小做了调整，当copy为1G时(copy=1L<<30)性能最佳，整过过程1-3秒左右。调至128M、512M。大约耗时都在15秒左右。为了公平起见，把其他方法中的byte缓冲区大小也做了同样调整，耗时变化不大。有些甚至更慢。
封装MappedBuffer工具类，代码如下：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
package com.boot.demo.test.io;
import java.io.Closeable;
import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
/**
* @author braska
* @date 2020/3/20
**/
public class MappedByteBufferReader implements Closeable {
private static final int BUFFERED_SIZE = 1 << 27;
private static final FileChannel.MapMode MAP_MODE = FileChannel.MapMode.READ_ONLY;
private FileChannel fileChannel;
private final long fileSize;
private final int bufferedSize;
BlockingQueue
public MappedByteBufferReader(String file) throws Exception {
this(file, BUFFERED_SIZE, MAP_MODE);
}
public MappedByteBufferReader(String file, int bufferedSize) throws Exception {
this(file, bufferedSize, MAP_MODE);
}
public MappedByteBufferReader(String file, FileChannel.MapMode mapMode) throws Exception {
this(file, BUFFERED_SIZE, mapMode);
}
public MappedByteBufferReader(String file, int bufferedSize, FileChannel.MapMode mapMode) throws Exception {
this.fileChannel = FileChannel.open(Paths.get(file));
this.fileSize = fileChannel.size();
this.bufferedSize = bufferedSize;
int capacity = (int) Math.ceil((double) fileSize / (double) bufferedSize);
this.queue = new ArrayBlockingQueue(capacity);
long readSize = bufferedSize;
long cursor = 0l;
while (cursor < fileSize) {
readSize = cursor + readSize > fileSize ? fileSize - cursor : readSize;
queue.add(
fileChannel.map(mapMode, cursor, readSize)
);
cursor += readSize;
}
}
public byte[] read() {
byte[] bytes;
MappedByteBuffer byteBuffer = queue.poll();
if (byteBuffer != null) {
int limit = byteBuffer.limit();
int position = byteBuffer.position();
int realSize = this.bufferedSize;
if (limit - position < this.bufferedSize) {
realSize = limit - position;
}
bytes = new byte[realSize];
byteBuffer.get(bytes);
byteBuffer.clear();
return bytes;
}
return null;
}
public long size() {
return this.fileSize;
}
@Override
public void close() throws IOException {
if (this.fileChannel != null) {
this.fileChannel.close();
}
}
public static void main(String[] args) {
long start = System.currentTimeMillis();
try (MappedByteBufferReader reader = new MappedByteBufferReader("E:\\software\\software.zip", 1 << 30);
FileChannel wc = FileChannel.open(Paths.get("E:\\software\\software_reader.zip"),
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE)) {
byte[] data;
MappedByteBuffer writer = wc.map(FileChannel.MapMode.READ_WRITE, 0, reader.size());
while ((data = reader.read()) != null) {
writer.put(data);
}
writer.clear();
System.out.println("used times: " + (System.currentTimeMillis() - start));
} catch (Exception e) {
e.printStackTrace();
}
}
}