Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

okhttp: add full implementation of HPACK header compression #6026

Merged
merged 4 commits into from Oct 8, 2019
Merged

okhttp: add full implementation of HPACK header compression #6026

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9b1024a
okhttp: added full implementation of HPACK header compression.
voidzcy Jul 30, 2019
79de730
added tests based on okhttp3's HpackTest
voidzcy Jul 30, 2019
0a0e751
Fixed some nits.
voidzcy Aug 19, 2019
83b3a9f
Disable Huffman encoding for default.
voidzcy Oct 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions okhttp/third_party/okhttp/main/java/io/grpc/okhttp/internal/framed/Header.java
View file
Open in desktop
Expand Up @@ -9,6 +9,8 @@
/** HTTP header: the name is an ASCII string, but the value can be UTF-8. */
public final class Header {
// Special header names defined in the SPDY and HTTP/2 specs.
static final ByteString PSEUDO_PREFIX = ByteString.encodeUtf8(":");
voidzcy marked this conversation as resolved.
Show resolved Hide resolved

public static final ByteString RESPONSE_STATUS = ByteString.encodeUtf8(":status");
public static final ByteString TARGET_METHOD = ByteString.encodeUtf8(":method");
public static final ByteString TARGET_PATH = ByteString.encodeUtf8(":path");
Expand Down
212 changes: 198 additions & 14 deletions okhttp/third_party/okhttp/main/java/io/grpc/okhttp/internal/framed/Hpack.java
View file
Open in desktop
Expand Up @@ -26,7 +26,6 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import okio.Buffer;
import okio.BufferedSource;
import okio.ByteString;
Expand All @@ -48,6 +47,14 @@ final class Hpack {
private static final int PREFIX_6_BITS = 0x3f;
private static final int PREFIX_7_BITS = 0x7f;

private static final int SETTINGS_HEADER_TABLE_SIZE = 4_096;

/**
* The decoder has ultimate control of the maximum size of the dynamic table but we can choose
* to use less. We'll put a cap at 16K. This is arbitrary but should be enough for most purposes.
*/
private static final int SETTINGS_HEADER_TABLE_SIZE_LIMIT = 16_384;

private static final io.grpc.okhttp.internal.framed.Header[] STATIC_HEADER_TABLE = new io.grpc.okhttp.internal.framed.Header[] {
new io.grpc.okhttp.internal.framed.Header(io.grpc.okhttp.internal.framed.Header.TARGET_AUTHORITY, ""),
new io.grpc.okhttp.internal.framed.Header(io.grpc.okhttp.internal.framed.Header.TARGET_METHOD, "GET"),
Expand Down Expand Up @@ -131,8 +138,13 @@ static final class Reader {
int dynamicTableByteCount = 0;

Reader(int headerTableSizeSetting, Source source) {
this(headerTableSizeSetting, headerTableSizeSetting, source);
}

// Visible for testing.
Reader(int headerTableSizeSetting, int maxDynamicTableByteCount, Source source) {
this.headerTableSizeSetting = headerTableSizeSetting;
this.maxDynamicTableByteCount = headerTableSizeSetting;
this.maxDynamicTableByteCount = maxDynamicTableByteCount;
this.source = Okio.buffer(source);
}

Expand Down Expand Up @@ -270,11 +282,15 @@ private void readLiteralHeaderWithIncrementalIndexingNewName() throws IOExceptio
insertIntoDynamicTable(-1, new io.grpc.okhttp.internal.framed.Header(name, value));
}

private ByteString getName(int index) {
private ByteString getName(int index) throws IOException {
if (isStaticHeader(index)) {
return STATIC_HEADER_TABLE[index].name;
} else {
return dynamicTable[dynamicTableIndex(index - STATIC_HEADER_TABLE.length)].name;
int dynamicTableIndex = dynamicTableIndex(index - STATIC_HEADER_TABLE.length);
if (dynamicTableIndex < 0 || dynamicTableIndex >= dynamicTable.length) {
throw new IOException("Header index too large " + (index + 1));
}
return dynamicTable[dynamicTableIndex].name;
}
}

Expand Down Expand Up @@ -373,26 +389,105 @@ private static Map<ByteString, Integer> nameToFirstIndex() {

static final class Writer {
private final Buffer out;
private boolean useCompression;
// Visible for testing.
int headerTableSizeSetting;

/**
* In the scenario where the dynamic table size changes multiple times between transmission of
* header blocks, we need to keep track of the smallest value in that interval.
*/
private int smallestHeaderTableSizeSetting = Integer.MAX_VALUE;
private boolean emitDynamicTableSizeUpdate = false;
private int maxDynamicTableByteCount;

// Visible for testing.
io.grpc.okhttp.internal.framed.Header[] dynamicTable = new io.grpc.okhttp.internal.framed.Header[8];
// Array is populated back to front, so new entries always have lowest index.
int nextDynamicTableIndex = dynamicTable.length - 1;
voidzcy marked this conversation as resolved.
Show resolved Hide resolved
int dynamicTableHeaderCount = 0;
voidzcy marked this conversation as resolved.
Show resolved Hide resolved
voidzcy marked this conversation as resolved.
Show resolved Hide resolved
voidzcy marked this conversation as resolved.
Show resolved Hide resolved
int dynamicTableByteCount = 0;

Writer(Buffer out) {
this(SETTINGS_HEADER_TABLE_SIZE, true, out);
}

// Visible for testing.
Writer(int headerTableSizeSetting, boolean useCompression, Buffer out) {
this.headerTableSizeSetting = headerTableSizeSetting;
this.maxDynamicTableByteCount = headerTableSizeSetting;
this.useCompression = useCompression;
this.out = out;
}

/** This does not use "never indexed" semantics for sensitive headers. */
// http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-12#section-6.2.3
void writeHeaders(List<io.grpc.okhttp.internal.framed.Header> headerBlock) throws IOException {
// TODO: implement index tracking
if (emitDynamicTableSizeUpdate) {
if (smallestHeaderTableSizeSetting < maxDynamicTableByteCount) {
// Multiple dynamic table size updates!
writeInt(smallestHeaderTableSizeSetting, PREFIX_5_BITS, 0x20);
}
emitDynamicTableSizeUpdate = false;
smallestHeaderTableSizeSetting = Integer.MAX_VALUE;
writeInt(maxDynamicTableByteCount, PREFIX_5_BITS, 0x20);
}

for (int i = 0, size = headerBlock.size(); i < size; i++) {
ByteString name = headerBlock.get(i).name.toAsciiLowercase();
io.grpc.okhttp.internal.framed.Header header = headerBlock.get(i);
ByteString name = header.name.toAsciiLowercase();
ByteString value = header.value;
int headerIndex = -1;
int headerNameIndex = -1;

Integer staticIndex = NAME_TO_FIRST_INDEX.get(name);
if (staticIndex != null) {
// Literal Header Field without Indexing - Indexed Name.
writeInt(staticIndex + 1, PREFIX_4_BITS, 0);
writeByteString(headerBlock.get(i).value);
} else {
out.writeByte(0x00); // Literal Header without Indexing - New Name.
headerNameIndex = staticIndex + 1;
if (headerNameIndex >= 2 && headerNameIndex <= 7) {
// Only search a subset of the static header table. Most entries have an empty value, so
// it's unnecessary to waste cycles looking at them. This check is built on the
// observation that the header entries we care about are in adjacent pairs, and we
// always know the first index of the pair.
if (STATIC_HEADER_TABLE[headerNameIndex - 1].value.equals(value)) {
headerIndex = headerNameIndex;
} else if (STATIC_HEADER_TABLE[headerNameIndex].value.equals(value)) {
headerIndex = headerNameIndex + 1;
}
}
}

if (headerIndex == -1) {
for (int j = nextDynamicTableIndex + 1; j < dynamicTable.length; j++) {
if (dynamicTable[j].name.equals(name)) {
if (dynamicTable[j].value.equals(value)) {
headerIndex = j - nextDynamicTableIndex + STATIC_HEADER_TABLE.length;
break;
} else if (headerNameIndex == -1) {
headerNameIndex = j - nextDynamicTableIndex + STATIC_HEADER_TABLE.length;
}
}
}
}

if (headerIndex != -1) {
// Indexed Header Field.
writeInt(headerIndex, PREFIX_7_BITS, 0x80);
} else if (headerNameIndex == -1) {
// Literal Header Field with Incremental Indexing - New Name.
out.writeByte(0x40);
writeByteString(name);
writeByteString(headerBlock.get(i).value);
writeByteString(value);
insertIntoDynamicTable(header);
} else if (name.startsWith(Header.PSEUDO_PREFIX) && !io.grpc.okhttp.internal.framed.Header.TARGET_AUTHORITY.equals(name)) {
// Follow Chromes lead - only include the :authority pseudo header, but exclude all other
// pseudo headers. Literal Header Field without Indexing - Indexed Name.
writeInt(headerNameIndex, PREFIX_4_BITS, 0);
writeByteString(value);
} else {
// Literal Header Field with Incremental Indexing - Indexed Name.
writeInt(headerNameIndex, PREFIX_6_BITS, 0x40);
writeByteString(value);
insertIntoDynamicTable(header);
}
}
}
Expand All @@ -419,8 +514,97 @@ void writeInt(int value, int prefixMask, int bits) throws IOException {
}

void writeByteString(ByteString data) throws IOException {
writeInt(data.size(), PREFIX_7_BITS, 0);
out.write(data);
if (useCompression && io.grpc.okhttp.internal.framed.Huffman.get().encodedLength(data.toByteArray()) < data.size()) {
Buffer huffmanBuffer = new Buffer();
io.grpc.okhttp.internal.framed.Huffman.get().encode(data.toByteArray(), huffmanBuffer.outputStream());
ByteString huffmanBytes = huffmanBuffer.readByteString();
writeInt(huffmanBytes.size(), PREFIX_7_BITS, 0x80);
out.write(huffmanBytes);
} else {
writeInt(data.size(), PREFIX_7_BITS, 0);
out.write(data);
}
}

int maxDynamicTableByteCount() {
return maxDynamicTableByteCount;
}

private void clearDynamicTable() {
Arrays.fill(dynamicTable, null);
nextDynamicTableIndex = dynamicTable.length - 1;
dynamicTableHeaderCount = 0;
dynamicTableByteCount = 0;
}

/** Returns the count of entries evicted. */
private int evictToRecoverBytes(int bytesToRecover) {
int entriesToEvict = 0;
if (bytesToRecover > 0) {
// determine how many headers need to be evicted.
for (int j = dynamicTable.length - 1; j >= nextDynamicTableIndex && bytesToRecover > 0; j--) {
bytesToRecover -= dynamicTable[j].hpackSize;
dynamicTableByteCount -= dynamicTable[j].hpackSize;
dynamicTableHeaderCount--;
entriesToEvict++;
}
System.arraycopy(dynamicTable, nextDynamicTableIndex + 1, dynamicTable,
nextDynamicTableIndex + 1 + entriesToEvict, dynamicTableHeaderCount);
nextDynamicTableIndex += entriesToEvict;
}
return entriesToEvict;
}

private void insertIntoDynamicTable(io.grpc.okhttp.internal.framed.Header entry) {
int delta = entry.hpackSize;

// if the new or replacement header is too big, drop all entries.
if (delta > maxDynamicTableByteCount) {
clearDynamicTable();
return;
}

// Evict headers to the required length.
int bytesToRecover = dynamicTableByteCount + delta - maxDynamicTableByteCount;
evictToRecoverBytes(bytesToRecover);

if (dynamicTableHeaderCount + 1 > dynamicTable.length) { // Need to grow the dynamic table.
io.grpc.okhttp.internal.framed.Header[] doubled = new io.grpc.okhttp.internal.framed.Header[dynamicTable.length * 2];
System.arraycopy(dynamicTable, 0, doubled, dynamicTable.length, dynamicTable.length);
nextDynamicTableIndex = dynamicTable.length - 1;
dynamicTable = doubled;
}
int index = nextDynamicTableIndex--;
dynamicTable[index] = entry;
dynamicTableHeaderCount++;
dynamicTableByteCount += delta;
}

void resizeHeaderTable(int headerTableSizeSetting) {
this.headerTableSizeSetting = headerTableSizeSetting;
int effectiveHeaderTableSize = Math.min(headerTableSizeSetting, SETTINGS_HEADER_TABLE_SIZE_LIMIT);

if (maxDynamicTableByteCount == effectiveHeaderTableSize) { // No change.
return;
}

if (effectiveHeaderTableSize < maxDynamicTableByteCount) {
smallestHeaderTableSizeSetting =
Math.min(smallestHeaderTableSizeSetting, effectiveHeaderTableSize);
}
emitDynamicTableSizeUpdate = true;
maxDynamicTableByteCount = effectiveHeaderTableSize;
adjustDynamicTableByteCount();
}

private void adjustDynamicTableByteCount() {
if (maxDynamicTableByteCount < dynamicTableByteCount) {
if (maxDynamicTableByteCount == 0) {
clearDynamicTable();
} else {
evictToRecoverBytes(dynamicTableByteCount - maxDynamicTableByteCount);
}
}
}
}

Expand Down