Source code

001/*
002 * ModeShape (http://www.modeshape.org)
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *       http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.modeshape.schematic.internal.document;
017
018import static org.junit.Assert.assertEquals;
019import static org.junit.Assert.assertNotNull;
020import static org.junit.Assert.fail;
021import java.io.ByteArrayInputStream;
022import java.io.ByteArrayOutputStream;
023import java.io.File;
024import java.io.FileInputStream;
025import java.io.IOException;
026import java.io.InputStream;
027import java.io.InputStreamReader;
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.Date;
031import java.util.LinkedHashMap;
032import java.util.List;
033import java.util.Map;
034import java.util.UUID;
035import java.util.concurrent.Callable;
036import java.util.concurrent.ExecutorService;
037import java.util.concurrent.Executors;
038import java.util.concurrent.Future;
039import java.util.concurrent.TimeUnit;
040import java.util.regex.Pattern;
041import org.bson.BSONObject;
042import org.bson.BasicBSONCallback;
043import org.bson.BasicBSONDecoder;
044import org.bson.BasicBSONEncoder;
045import org.bson.BasicBSONObject;
046import org.bson.types.BSONTimestamp;
047import org.bson.types.BasicBSONList;
048import org.codehaus.jackson.JsonToken;
049import org.junit.After;
050import org.junit.Assert;
051import org.junit.Before;
052import org.junit.Test;
053import org.modeshape.schematic.document.Binary;
054import org.modeshape.schematic.document.Code;
055import org.modeshape.schematic.document.CodeWithScope;
056import org.modeshape.schematic.document.Document;
057import org.modeshape.schematic.document.Json;
058import org.modeshape.schematic.document.MaxKey;
059import org.modeshape.schematic.document.MinKey;
060import org.modeshape.schematic.document.ObjectId;
061import org.modeshape.schematic.document.Symbol;
062import org.modeshape.schematic.document.Timestamp;
063import org.modeshape.schematic.internal.annotation.FixFor;
064import org.modeshape.schematic.internal.io.BufferCache;
065
066public class BsonReadingAndWritingTest {
067
068    protected BsonReader reader;
069    protected BsonWriter writer;
070    protected Document input;
071    protected Document output;
072    protected boolean print;
073
074    @Before
075    public void beforeTest() {
076        reader = new BsonReader();
077        writer = new BsonWriter();
078        print = false;
079    }
080
081    @After
082    public void afterTest() {
083        reader = null;
084        writer = null;
085    }
086
087    @Test
088    public void shouldReadExampleBsonStream() throws IOException {
089        // "\x16\x00\x00\x00\x02hello\x00\x06\x00\x00\x00world\x00\x00"
090        byte[] bytes = new byte[] {0x16, 0x00, 0x00, 0x00, 0x02, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x00, 0x06, 0x00, 0x00, 0x00,
091            0x77, 0x6f, 0x72, 0x6c, 0x64, 0x00, 0x00};
092        output = reader.read(new ByteArrayInputStream(bytes));
093        String json = Json.write(output);
094        String expected = "{ \"hello\" : \"world\" }";
095        if (print) {
096            System.out.println(json);
097            System.out.flush();
098        }
099        assertEquals(expected, json);
100    }
101
102    @Test
103    public void shouldRoundTripSimpleBsonObjectWithStringValue() {
104        input = new BasicDocument("name", "Joe");
105        assertRoundtrip(input);
106    }
107
108    @Test
109    public void shouldRoundTripSimpleBsonObjectWithBooleanValue() {
110        input = new BasicDocument("foo", 3L);
111        assertRoundtrip(input);
112    }
113
114    @Test
115    public void shouldRoundTripSimpleBsonObjectWithIntValue() {
116        input = new BasicDocument("foo", 3);
117        assertRoundtrip(input);
118    }
119
120    @Test
121    public void shouldRoundTripSimpleBsonObjectWithLongValue() {
122        input = new BasicDocument("foo", 3L);
123        assertRoundtrip(input);
124    }
125
126    @Test
127    public void shouldRoundTripSimpleBsonObjectWithFloatValue() {
128        input = new BasicDocument("foo", 3.0f);
129        assertRoundtrip(input);
130    }
131
132    @Test
133    public void shouldRoundTripSimpleBsonObjectWithDoubleValue() {
134        input = new BasicDocument("foo", 3.0d);
135        assertRoundtrip(input);
136    }
137
138    @Test
139    public void shouldRoundTripSimpleBsonObjectWithDateValue() {
140        input = new BasicDocument("foo", new Date());
141        assertRoundtrip(input);
142    }
143
144    @Test
145    public void shouldRoundTripSimpleBsonObjectWithTimestampValue() {
146        input = new BasicDocument("foo", new Timestamp(new Date()));
147        assertRoundtrip(input);
148    }
149
150    @Test
151    public void shouldRoundTripSimpleBsonObjectWithObjectId() {
152        // print = true;
153        int time = Math.abs((int) new Date().getTime());
154        if (print) System.out.println("time value: " + time);
155        input = new BasicDocument("foo", new ObjectId(time, 1, 2, 3));
156        assertRoundtrip(input);
157    }
158
159    @Test
160    public void shouldRoundTripSimpleBsonObjectWithCode() {
161        input = new BasicDocument("foo", new Code("bar"));
162        assertRoundtrip(input);
163    }
164
165    @Test
166    public void shouldRoundTripSimpleBsonObjectWithCodeWithScope() {
167        Document scope = new BasicDocument("baz", "bam", "bak", "bat");
168        input = new BasicDocument("foo", new CodeWithScope("bar", scope));
169        assertRoundtrip(input);
170    }
171
172    @Test
173    public void shouldRoundTripSimpleBsonObjectWithMaxKey() {
174        input = new BasicDocument("foo", MaxKey.getInstance());
175        assertRoundtrip(input, false);
176    }
177
178    @Test
179    public void shouldRoundTripSimpleBsonObjectWithMinKey() {
180        input = new BasicDocument("foo", MinKey.getInstance());
181        assertRoundtrip(input, false);
182    }
183
184    @Test
185    public void shouldRoundTripSimpleBsonObjectWithSymbol() {
186        input = new BasicDocument("foo", new Symbol("bar"));
187        assertRoundtrip(input);
188    }
189
190    @Test
191    public void shouldRoundTripSimpleBsonObjectWithNull() {
192        input = new BasicDocument("foo", null);
193        assertRoundtrip(input);
194    }
195
196    @Test
197    public void shouldRoundTripSimpleBsonObjectWithBinary1() {
198        byte[] data = new byte[] {0x16, 0x00, 0x00, 0x00, 0x02, 0x68, 0x65, 0x6c};
199        input = new BasicDocument("foo", new Binary(data));
200        assertRoundtrip(input);
201    }
202
203    @Test
204    //Fix-For MODE-1575
205    public void shouldRoundTripSimpleBsonObjectWithBinary2() throws Exception {
206        ByteArrayOutputStream bos = new ByteArrayOutputStream();
207        InputStream is = getClass().getClassLoader().getResourceAsStream("binary");
208        assertNotNull(is);
209        try {
210            byte[] buff = new byte[1024];
211            int read;
212            while ((read = is.read(buff)) != -1) {
213                bos.write(buff, 0, read);
214            }
215        } finally {
216            bos.close();
217            is.close();
218        }
219
220        input = new BasicDocument("foo", new Binary(bos.toByteArray()));
221        assertRoundtrip(input);
222    }
223
224    @Test
225    public void shouldRoundTripSimpleBsonObjectWithUuid() {
226        input = new BasicDocument("foo", UUID.randomUUID());
227        assertRoundtrip(input);
228    }
229
230    @Test
231    public void shouldRoundTripSimpleBsonObjectWithPattern() {
232        // print = true;
233        input = new BasicDocument("foo", Pattern.compile("[CH]at\\s+"));
234        assertRoundtrip(input);
235    }
236
237    @Test
238    public void shouldRoundTripSimpleBsonObjectWithPatternAndFlags() {
239        // print = true;
240        input = new BasicDocument("foo", Pattern.compile("[CH]at\\s+", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE));
241        assertRoundtrip(input);
242    }
243
244    @Test
245    public void shouldRoundTripSimpleBsonObjectWithArray() {
246        BasicArray array = new BasicArray();
247        array.addValue("value1");
248        array.addValue(new Symbol("value2"));
249        array.addValue(30);
250        array.addValue(40L);
251        array.addValue(4.33d);
252        array.addValue(false);
253        array.addValue(null);
254        array.addValue("value2");
255        input = new BasicDocument("foo", array);
256        assertRoundtrip(input);
257    }
258
259    @Test
260    public void shouldRoundTripBsonObjectWithTwoFields() {
261        input = new BasicDocument("name", "Joe", "age", 35);
262        assertRoundtrip(input);
263    }
264
265    @Test
266    public void shouldRoundTripBsonObjectWithThreeFields() {
267        input = new BasicDocument("name", "Joe", "age", 35, "nick", "joey");
268        assertRoundtrip(input);
269    }
270
271    @Test
272    public void shouldRoundTripBsonObjectWithNestedDocument() {
273        BasicDocument address = new BasicDocument("street", "100 Main", "city", "Springfield", "zip", 12345);
274        input = new BasicDocument("name", "Joe", "age", 35, "address", address, "nick", "joey");
275        assertRoundtrip(input);
276    }
277
278    @Test
279    public void shouldRoundTripLargeModeShapeDocument() throws Exception {
280        Document doc = Json.read(resource("json/sample-large-modeshape-doc.json"));
281        // OutputStream os = new FileOutputStream("src/test/resources/json/sample-large-modeshape-doc2.json");
282        // Json.writePretty(doc, os);
283        // os.flush();
284        // os.close();
285        assertRoundtrip(doc);
286    }
287
288    @Test
289    @FixFor( "MODE-2074" )
290    public void shouldRoundTripBsonWithLargeStringField() throws Exception {
291        //use a string which overflows the default buffer BufferCache.MINIMUM_SIZE
292        final String largeString = readFile("json/sample-large-modeshape-doc3.json");
293        Document document = new BasicDocument("largeString", largeString);
294        assertRoundtrip(document);
295    }
296
297    @Test
298    @FixFor( "MODE-2074" )
299    public void shouldRoundTripBsonWithLargeStringFieldFromMultipleThreads() throws Exception {
300        final String largeString = readFile("json/sample-large-modeshape-doc3.json");
301        int threadCount = 10;
302        List<Future<Void>> results = new ArrayList<Future<Void>>();
303        ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
304        for (int i = 0; i < threadCount; i++) {
305            results.add(executorService.submit(new Callable<Void>() {
306                @Override
307                public Void call() throws Exception {
308                    Document document = new BasicDocument("largeString", largeString);
309                    assertRoundtrip(document);
310                    return null;
311                }
312            }));
313        }
314
315        for (Future<Void> result : results) {
316            result.get(1, TimeUnit.SECONDS);
317        }
318    }
319    
320    @Test
321    @FixFor( "MODE-2430" )
322    public void shouldRoundTripLargeStringsSuccessively() throws Exception {
323        int limit = 1048 * 8; // the default buffer size
324        int iterations = 20;
325        char letter = 'a';
326        for (int i = 0; i < iterations; i++) {
327            int size = limit + i;
328            char[] chars = new char[size];
329            Arrays.fill(chars, letter);
330            letter = (char)((byte) letter + 1);
331            String str = new String(chars);
332            Document document = new BasicDocument("largeString", str);
333            assertRoundtrip(document, false);
334        }
335    }
336
337    @Test
338    @FixFor( "MODE-2615" )
339    public void shouldRoundTripDocumentWithMultiByteUTF8Chars() throws Exception{
340        char[] chars = new char[BufferCache.MINIMUM_SIZE];
341        Arrays.fill(chars, 'a');
342        chars[BufferCache.MINIMUM_SIZE - 1] = '\u00A3'; // 2 bytes UTF-8
343        Document document = new BasicDocument("string", new String(chars));
344        assertRoundtrip(document);
345
346        chars[BufferCache.MINIMUM_SIZE - 1] = '\uFFFF'; // 3 bytes UTF-8
347        document = new BasicDocument("string", new String(chars));
348        assertRoundtrip(document);
349
350        chars[BufferCache.MINIMUM_SIZE - 1] = '\u00A3'; // 2 bytes UTF-8
351        chars[BufferCache.MINIMUM_SIZE - 2] = '\uFFFF'; // 3 bytes UTF-8
352        document = new BasicDocument("string", new String(chars));
353        assertRoundtrip(document);
354
355        chars[BufferCache.MINIMUM_SIZE - 1] = '\uFFFF'; // 3 bytes UTF-8
356        chars[BufferCache.MINIMUM_SIZE - 2] = '\u00A3'; // 2 bytes UTF-8
357        document = new BasicDocument("string", new String(chars));
358        assertRoundtrip(document);
359    }
360
361    protected String readFile(String filePath) throws IOException {
362        InputStreamReader reader = new InputStreamReader(resource(filePath));
363        StringBuilder stringBuilder = new StringBuilder();
364        boolean error = false;
365        try {
366            int numRead = 0;
367            char[] buffer = new char[1024];
368            while ((numRead = reader.read(buffer)) > -1) {
369                stringBuilder.append(buffer, 0, numRead);
370            }
371        } catch (IOException e) {
372            error = true; // this error should be thrown, even if there is an error closing reader
373            throw e;
374        } catch (RuntimeException e) {
375            error = true; // this error should be thrown, even if there is an error closing reader
376            throw e;
377        } finally {
378            try {
379                reader.close();
380            } catch (IOException e) {
381                if (!error) throw e;
382            }
383        }
384        return stringBuilder.toString();
385    }
386
387    protected void assertRoundtrip( Document input ) {
388        assertRoundtrip(input, true);
389    }
390
391    protected void assertRoundtrip( Document input,
392                                    boolean compareToOtherImpls ) {
393        assertNotNull(input);
394        Document output = writeThenRead(input, compareToOtherImpls);
395        if (print) {
396            System.out.println("********************************************************************************");
397            System.out.println("INPUT :  " + input);
398            System.out.println();
399            System.out.println("OUTPUT:  " + output);
400            System.out.println("********************************************************************************");
401            System.out.flush();
402        }
403        Assert.assertEquals("Round trip failed", input, output);
404    }
405
406    protected Document writeThenRead( Document object,
407                                      boolean compareToOtherImpls ) {
408        try {
409            long start = System.nanoTime();
410            byte[] bytes = writer.write(object);
411            long writeTime = System.nanoTime() - start;
412
413            start = System.nanoTime();
414            Document result = reader.read(new ByteArrayInputStream(bytes));
415            long readTime = System.nanoTime() - start;
416
417            if (compareToOtherImpls) {
418                // Convert to MongoDB, write to bytes, and compare ...
419                BSONObject mongoData = createMongoData(object);
420                start = System.nanoTime();
421                byte[] mongoBytes = new BasicBSONEncoder().encode(mongoData);
422                long mongoWriteTime = System.nanoTime() - start;
423                assertSame(bytes, mongoBytes, "BSON   ", "Mongo  ");
424
425                // FYI: The Jackson BSON library writes several of the types incorrectly,
426                // whereas the MongoDB library seems to write things per the spec.
427
428                // // Convert to Jackson BSON, write to bytes, and compare ...
429                // ByteArrayOutputStream stream2 = new ByteArrayOutputStream();
430                // ObjectMapper om = new ObjectMapper(new BsonFactory());
431                // Map<String, Object> jacksonData = createJacksonData(object);
432                // om.writeValue(stream2, jacksonData);
433                // byte[] jacksonBytes = stream2.toByteArray();
434                // assertSame(bytes, jacksonBytes, "BSON   ", "Jackson");
435
436                start = System.nanoTime();
437                new BasicBSONDecoder().decode(bytes, new BasicBSONCallback());
438                long mongoReadTime = System.nanoTime() - start;
439
440                Document fromMongo = reader.read(new ByteArrayInputStream(mongoBytes));
441                if (!fromMongo.equals(result)) {
442                    System.out.println("from Schematic: " + result);
443                    System.out.println("from Mongo:     " + fromMongo);
444                    fail("Document read from bytes written by Mongo did not match expected document: " + result);
445                }
446
447                if (print) {
448                    System.out.println("Reading with Schematic:  " + percent(readTime, mongoReadTime) + " than Mongo");
449                    System.out.println("Writing with Schematic:  " + percent(writeTime, mongoWriteTime) + " than Mongo");
450                }
451            }
452
453            return result;
454        } catch (IOException e) {
455            throw new AssertionError(e);
456        }
457    }
458
459    protected String time( long nanos ) {
460        return "" + TimeUnit.NANOSECONDS.convert(nanos, TimeUnit.NANOSECONDS) + "ns";
461    }
462
463    protected String percent( long nanos1,
464                              long nanos2 ) {
465        float percent = 100.0f * (float)(((double)nanos2 - (double)nanos1) / nanos1);
466        if (percent < 0.0d) {
467            return "" + -percent + "% slower";
468        }
469        return "" + percent + "% faster";
470    }
471
472    protected BSONObject createMongoData( Document document ) {
473        BSONObject obj = new BasicBSONObject();
474        for (Document.Field field : document.fields()) {
475            Object value = field.getValue();
476            obj.put(field.getName(), createMongoData(value));
477        }
478        return obj;
479    }
480
481    protected Object createMongoData( Object value ) {
482        if (value instanceof MinKey) {
483            value = "MinKey";
484        } else if (value instanceof MaxKey) {
485            value = "MaxKey";
486        } else if (value instanceof Symbol) {
487            Symbol symbol = (Symbol)value;
488            value = new org.bson.types.Symbol(symbol.getSymbol());
489        } else if (value instanceof ObjectId) {
490            ObjectId id = (ObjectId)value;
491            value = new org.bson.types.ObjectId(id.getBytes());
492        } else if (value instanceof Timestamp) {
493            Timestamp ts = (Timestamp)value;
494            value = new BSONTimestamp(ts.getTime(), ts.getInc());
495        } else if (value instanceof CodeWithScope) {
496            CodeWithScope code = (CodeWithScope)value;
497            value = new org.bson.types.CodeWScope(code.getCode(), createMongoData(code.getScope()));
498        } else if (value instanceof Code) {
499            Code code = (Code)value;
500            value = new org.bson.types.Code(code.getCode());
501        } else if (value instanceof Binary) {
502            Binary binary = (Binary)value;
503            value = new org.bson.types.Binary(binary.getBytes());
504        } else if (value instanceof List) {
505            List<?> values = (List<?>)value;
506            BasicBSONList newValues = new BasicBSONList();
507            for (Object v : values) {
508                newValues.add(createMongoData(v));
509            }
510            value = newValues;
511        } else if (value instanceof Document) {
512            value = createMongoData((Document)value);
513        }
514        return value;
515    }
516
517    protected Map<String, Object> createJacksonData( Document document ) {
518        Map<String, Object> data = new LinkedHashMap<String, Object>();
519        for (Document.Field field : document.fields()) {
520            Object value = field.getValue();
521            data.put(field.getName(), createJacksonData(value));
522        }
523        return data;
524    }
525
526    protected Object createJacksonData( Object value ) {
527        if (value instanceof MinKey) {
528            value = JsonToken.VALUE_STRING;
529        } else if (value instanceof MaxKey) {
530            value = JsonToken.VALUE_STRING;
531        } else if (value instanceof Symbol) {
532            value = new de.undercouch.bson4jackson.types.Symbol(((Symbol)value).getSymbol());
533        } else if (value instanceof ObjectId) {
534            ObjectId id = (ObjectId)value;
535            value = new de.undercouch.bson4jackson.types.ObjectId(id.getTime(), id.getMachine(), id.getInc());
536        } else if (value instanceof Timestamp) {
537            Timestamp ts = (Timestamp)value;
538            value = new de.undercouch.bson4jackson.types.Timestamp(ts.getTime(), ts.getInc());
539        } else if (value instanceof CodeWithScope) {
540            CodeWithScope code = (CodeWithScope)value;
541            value = new de.undercouch.bson4jackson.types.JavaScript(code.getCode(), createJacksonData(code.getScope()));
542        } else if (value instanceof Code) {
543            Code code = (Code)value;
544            value = new de.undercouch.bson4jackson.types.JavaScript(code.getCode(), null);
545        } else if (value instanceof List) {
546            List<?> values = (List<?>)value;
547            List<Object> newValues = new ArrayList<Object>(values.size());
548            for (Object v : values) {
549                newValues.add(createJacksonData(v));
550            }
551            value = newValues;
552        } else if (value instanceof Document) {
553            value = createJacksonData((Document)value);
554        }
555        return value;
556    }
557
558    protected void assertSame( byte[] b1,
559                               byte[] b2,
560                               String name1,
561                               String name2 ) {
562        if (b1.equals(b2)) return;
563        int s1 = b1.length;
564        int s2 = b2.length;
565        String sb1 = toString(b1);
566        String sb2 = toString(b2);
567        if (!sb1.equals(sb2)) {
568            System.out.println(name1 + " size: " + padLeft(s1, 3) + " content: " + sb1);
569            System.out.println(name2 + " size: " + padLeft(s2, 3) + " content: " + sb2);
570            fail();
571        }
572    }
573
574    protected String padLeft( Object value,
575                              int width ) {
576        String result = value != null ? value.toString() : "null";
577        while (result.length() < width) {
578            result = " " + result;
579        }
580        return result;
581    }
582
583    protected String toString( byte[] bytes ) {
584        StringBuilder sb = new StringBuilder();
585        for (byte b : bytes) {
586            sb.append(padLeft((int)b, 4)).append(' ');
587        }
588        return sb.toString();
589    }
590
591    protected boolean delete( File fileOrDirectory ) {
592        if (fileOrDirectory == null) {
593            return false;
594        }
595        if (!fileOrDirectory.exists()) {
596            return false;
597        }
598
599        // The file/directory exists, so if a directory delete all of the contents ...
600        if (fileOrDirectory.isDirectory()) {
601            for (File childFile : fileOrDirectory.listFiles()) {
602                delete(childFile); // recursive call (good enough for now until we need something better)
603            }
604            // Now an empty directory ...
605        }
606        // Whether this is a file or empty directory, just delete it ...
607        return fileOrDirectory.delete();
608    }
609
610    protected InputStream resource( String resourcePath ) {
611        InputStream stream = BsonReadingAndWritingTest.class.getClassLoader().getResourceAsStream(resourcePath);
612        if (stream == null) {
613            File file = new File(resourcePath);
614            if (!file.exists()) {
615                file = new File("src/test/resources" + resourcePath);
616            }
617            if (!file.exists()) {
618                file = new File("src/test/resources/" + resourcePath);
619            }
620            if (file.exists()) {
621                try {
622                    stream = new FileInputStream(file);
623                } catch (IOException e) {
624                    throw new AssertionError("Failed to open stream to \"" + file.getAbsolutePath() + "\"");
625                }
626            }
627        }
628        assert stream != null : "Resource at \"" + resourcePath + "\" could not be found";
629        return stream;
630    }
631
632}