Ensure unicode characters are interpreted as desired (closes #20)

Author: TiganeteaRobert

analytics/shred.go

@@ -15,17 +15,14 @@ package analytics
 
 import (
 	"fmt"
+	"github.com/pkg/errors"
 	"regexp"
 	"strings"
 	"unicode" // For camel to snake case - consider alternative?
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/pkg/errors"
 )
 
-var ConfigCompatibleWithStandardLibrary = jsoniter.Config{EscapeHTML: false}
-var json = jsoniter.ConfigCompatibleWithStandardLibrary
-
 type SelfDescribingData struct {
 	Schema string
 	Data   map[string]interface{} // TODO: See if leaving data as a string or byte array would work, and would be faster.
@@ -103,7 +100,7 @@ func fixSchema(prefix string, schemaUri string) (string, error) {
 func shredContexts(contexts string) ([]KeyVal, error) {
 	ctxts := Contexts{}
 
-	err := json.Unmarshal([]byte(contexts), &ctxts)
+	err := jsoniter.Unmarshal([]byte(contexts), &ctxts)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error unmarshaling context JSON")
 	}
@@ -137,7 +134,7 @@ func shredUnstruct(unstruct string) ([]KeyVal, error) {
 
 	event := UnstructEvent{}
 
-	err := json.Unmarshal([]byte(unstruct), &event)
+	err := jsoniter.Unmarshal([]byte(unstruct), &event)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error unmarshaling unstruct event JSON")
 	}

analytics/transform.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 )
 
@@ -27,6 +28,8 @@ const (
 	EmptyFieldErr string = `Field is empty`
 )
 
+var json = jsoniter.Config{}.Froze()
+
 type KeyVal struct {
 	Key   string
 	Value interface{}

analytics/transform_test.go

@@ -14,9 +14,9 @@
 package analytics
 
 import (
-	stdJson "encoding/json" // Using the std JSON package for expected values
 	"testing"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -201,14 +201,12 @@ func BenchmarkMapifyGoodEvent(b *testing.B) {
 func TestToJson(t *testing.T) {
 	assert := assert.New(t)
 
-	// correct value
-	jsonEvent, err := stdJson.Marshal(eventMapWithoutGeo)
-	if err != nil {
-	}
-
 	jsonifiedEvent, err := fullEvent.ToJson()
 	assert.Nil(err)
-	assert.Equal(jsonEvent, jsonifiedEvent)
+
+	jsonifiedEventOrdered, _ := orderJson(jsonifiedEvent)
+
+	assert.Equal(eventMapWithoutGeoJSON, jsonifiedEventOrdered)
 
 	// incorrect input
 	failedJsonify, err := ParsedEvent([]string{"one", "two"}).ToJson()
@@ -222,17 +220,30 @@ func BenchmarkToJson(b *testing.B) {
 	}
 }
 
-func TestToJsonWithGeo(t *testing.T) {
-	assert := assert.New(t)
-
-	// correct value
-	jsonEvent, err := stdJson.Marshal(eventMapWithGeo)
+// orderJson uses the jsoniter library to remarshal a []byte, resulting in a sorted JSON
+func orderJson(bytes []byte) ([]byte, error) {
+	var ifce interface{}
+	j := jsoniter.Config{SortMapKeys: true}.Froze()
+	err := j.Unmarshal(bytes, &ifce)
+	if err != nil {
+		return []byte{}, err
+	}
+	output, err := j.Marshal(ifce)
 	if err != nil {
+		return []byte{}, err
 	}
+	return output, nil
+}
+
+func TestToJsonWithGeo(t *testing.T) {
+	assert := assert.New(t)
 
 	jsonifiedEvent, err := fullEvent.ToJsonWithGeo()
 	assert.Nil(err)
-	assert.Equal(jsonEvent, jsonifiedEvent)
+
+	jsonifiedEventOrdered, _ := orderJson(jsonifiedEvent)
+
+	assert.Equal(eventMapWithGeoJSON, jsonifiedEventOrdered)
 
 	// incorrect input
 	failedJsonify, err := ParsedEvent([]string{"one", "two"}).ToJsonWithGeo()
@@ -330,6 +341,10 @@ func TestGetUnstructEventValue(t *testing.T) {
 	unstructValue, err = fullEvent.GetUnstructEventValue(`elementId`)
 	assert.Nil(err)
 	assert.Equal(`exampleLink`, unstructValue)
+
+	unstructValue, err = fullEvent.GetUnstructEventValue(`unicodeTest`)
+	assert.Nil(err)
+	assert.Equal(`<>angry_birds`, unstructValue)
 }
 
 func TestGetContextValue(t *testing.T) {
@@ -394,12 +409,19 @@ func TestGetSubsetJSON(t *testing.T) {
 
 	// correct value
 	subsetJsonValue, err := fullEvent.GetSubsetJson([]string{"app_id", "br_features_flash", "br_features_pdf", "collector_tstamp", "unstruct_event", "contexts", "derived_contexts"}...)
-	assert.Equal(subsetJson, subsetJsonValue)
+
+	subsetJsonOrdered, _ := orderJson(subsetJson)
+	subsetJsonValueOrdered, _ := orderJson(subsetJsonValue)
+
+	assert.Equal(subsetJsonOrdered, subsetJsonValueOrdered)
 	assert.Nil(err)
 
 	// correct values passing multiple string args
 	subsetJsonValue2, err := fullEvent.GetSubsetJson("app_id", "br_features_flash", "br_features_pdf", "collector_tstamp", "unstruct_event", "contexts", "derived_contexts")
-	assert.Equal(subsetJson, subsetJsonValue2)
+
+	subsetJsonValue2Ordered, _ := orderJson(subsetJsonValue2)
+
+	assert.Equal(subsetJsonOrdered, subsetJsonValue2Ordered)
 	assert.Nil(err)
 
 	// incorrect field name
@@ -408,7 +430,7 @@ func TestGetSubsetJSON(t *testing.T) {
 	assert.NotNil(err)
 
 	// empty value
-	emptyJson, _ := stdJson.Marshal(make(map[string]interface{}))
+	emptyJson, _ := jsoniter.Marshal(make(map[string]interface{}))
 	emptyValue, err := fullEvent.GetSubsetJson("ti_name")
 	assert.Equal(emptyJson, emptyValue)
 	assert.Nil(err)

analytics/vars_test.go

@@ -14,9 +14,10 @@
 package analytics
 
 import (
-	stdJson "encoding/json"
 	"strings"
 	"time"
+
+	jsoniter "github.com/json-iterator/go"
 )
 
 var ctxt = `{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:com.acme/test_context/jsonschema/1-0-0","data":{"field1": 1}}, {"schema":"iglu:com.acme/test_context/jsonschema/1-0-0","data":{"field1": 2}}]}`
@@ -27,7 +28,7 @@ var invalidUnstruct = `{"data":{"data":{"key":"value"},"schema":"fail"},"schema"
 
 var tstampValue, _ = time.Parse("2006-01-02 15:04:05.999", "2013-11-26 00:03:57.885")
 
-var unstructString = `{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/link_click/jsonschema/1-0-1","data":{"targetUrl":"http://www.example.com","elementClasses":["foreground"],"elementId":"exampleLink"}}}`
+var unstructString = `{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/link_click/jsonschema/1-0-1","data":{"targetUrl":"http://www.example.com","elementClasses":["foreground"],"elementId":"exampleLink","unicodeTest":"<>angry_birds"}}}`
 
 var contextsString = `{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:org.schema/WebPage/jsonschema/1-0-0","data":{"genre":"blog","inLanguage":"en-US","datePublished":"2014-11-06T00:00:00Z","author":"Fred Blundun","breadcrumb":["blog","releases"],"keywords":["snowplow","javascript","tracker","event"]}},{"schema":"iglu:org.w3/PerformanceTiming/jsonschema/1-0-0","data":{"navigationStart":1415358089861,"unloadEventStart":1415358090270,"unloadEventEnd":1415358090287,"redirectStart":0,"redirectEnd":0,"fetchStart":1415358089870,"domainLookupStart":1415358090102,"domainLookupEnd":1415358090102,"connectStart":1415358090103,"connectEnd":1415358090183,"requestStart":1415358090183,"responseStart":1415358090265,"responseEnd":1415358090265,"domLoading":1415358090270,"domInteractive":1415358090886,"domContentLoadedEventStart":1415358090968,"domContentLoadedEventEnd":1415358091309,"domComplete":0,"loadEventStart":0,"loadEventEnd":0}}]}`
 
@@ -171,6 +172,9 @@ var fullEvent = ParsedEvent([]string{
 // tsv string
 var tsvEvent = strings.Join(fullEvent, "\t")
 
+var eventMapWithGeoJSON = []byte(`{"app_id":"<>angry-birds","br_features_flash":false,"br_features_pdf":true,"collector_tstamp":"2013-11-26T00:03:57.885Z","contexts_com_snowplowanalytics_snowplow_ua_parser_context_1":[{"deviceFamily":"Other","osFamily":"Windows XP","osMajor":null,"osMinor":null,"osPatch":null,"osPatchMinor":null,"osVersion":"Windows XP","useragentFamily":"IE","useragentMajor":"7","useragentMinor":"0","useragentPatch":null,"useragentVersion":"IE 7.0"}],"contexts_org_schema_web_page_1":[{"author":"Fred Blundun","breadcrumb":["blog","releases"],"datePublished":"2014-11-06T00:00:00Z","genre":"blog","inLanguage":"en-US","keywords":["snowplow","javascript","tracker","event"]}],"contexts_org_w3_performance_timing_1":[{"connectEnd":1415358090183,"connectStart":1415358090103,"domComplete":0,"domContentLoadedEventEnd":1415358091309,"domContentLoadedEventStart":1415358090968,"domInteractive":1415358090886,"domLoading":1415358090270,"domainLookupEnd":1415358090102,"domainLookupStart":1415358090102,"fetchStart":1415358089870,"loadEventEnd":0,"loadEventStart":0,"navigationStart":1415358089861,"redirectEnd":0,"redirectStart":0,"requestStart":1415358090183,"responseEnd":1415358090265,"responseStart":1415358090265,"unloadEventEnd":1415358090287,"unloadEventStart":1415358090270}],"derived_tstamp":"2013-11-26T00:03:57.885Z","domain_sessionid":"2b15e5c8-d3b1-11e4-b9d6-1681e6b88ec1","domain_sessionidx":3,"domain_userid":"bc2e92ec6c204a14","dvce_created_tstamp":"2013-11-26T00:03:57.885Z","etl_tstamp":"2013-11-26T00:03:57.885Z","event":"page_view","event_fingerprint":"e3dbfa9cca0412c3d4052863cefb547f","event_format":"jsonschema","event_id":"c6ef3124-b53a-4b13-a233-0088f79dcbcb","event_name":"link_click","event_vendor":"com.snowplowanalytics.snowplow","event_version":"1-0-0","geo_city":"New York","geo_country":"US","geo_latitude":37.443604,"geo_location":"37.443604,-122.4124","geo_longitude":-122.4124,"geo_region":"TX","geo_region_name":"Florida","geo_zipcode":"94109","ip_domain":"nuvox.net","ip_isp":"FDN Communications","ip_netspeed":"Cable/DSL","ip_organization":"Bouygues Telecom","name_tracker":"cloudfront-1","network_userid":"ecdff4d0-9175-40ac-a8bb-325c49733607","page_title":"On Analytics","page_url":"http://www.snowplowanalytics.com","page_urlfragment":"4-conclusion","page_urlhost":"www.snowplowanalytics.com","page_urlpath":"/product/index.html","page_urlport":80,"page_urlquery":"id=GTM-DLRG","page_urlscheme":"http","platform":"web","true_tstamp":"2013-11-26T00:03:57.885Z","txn_id":41828,"unstruct_event_com_snowplowanalytics_snowplow_link_click_1":{"elementClasses":["foreground"],"elementId":"exampleLink","targetUrl":"http://www.example.com","unicodeTest":"<>angry_birds"},"user_fingerprint":"2161814971","user_id":"[email protected]","user_ipaddress":"92.231.54.234","v_collector":"clj-tomcat-0.1.0","v_etl":"serde-0.5.2","v_tracker":"js-2.1.0"}`)
+var eventMapWithoutGeoJSON = []byte(`{"app_id":"<>angry-birds","br_features_flash":false,"br_features_pdf":true,"collector_tstamp":"2013-11-26T00:03:57.885Z","contexts_com_snowplowanalytics_snowplow_ua_parser_context_1":[{"deviceFamily":"Other","osFamily":"Windows XP","osMajor":null,"osMinor":null,"osPatch":null,"osPatchMinor":null,"osVersion":"Windows XP","useragentFamily":"IE","useragentMajor":"7","useragentMinor":"0","useragentPatch":null,"useragentVersion":"IE 7.0"}],"contexts_org_schema_web_page_1":[{"author":"Fred Blundun","breadcrumb":["blog","releases"],"datePublished":"2014-11-06T00:00:00Z","genre":"blog","inLanguage":"en-US","keywords":["snowplow","javascript","tracker","event"]}],"contexts_org_w3_performance_timing_1":[{"connectEnd":1415358090183,"connectStart":1415358090103,"domComplete":0,"domContentLoadedEventEnd":1415358091309,"domContentLoadedEventStart":1415358090968,"domInteractive":1415358090886,"domLoading":1415358090270,"domainLookupEnd":1415358090102,"domainLookupStart":1415358090102,"fetchStart":1415358089870,"loadEventEnd":0,"loadEventStart":0,"navigationStart":1415358089861,"redirectEnd":0,"redirectStart":0,"requestStart":1415358090183,"responseEnd":1415358090265,"responseStart":1415358090265,"unloadEventEnd":1415358090287,"unloadEventStart":1415358090270}],"derived_tstamp":"2013-11-26T00:03:57.885Z","domain_sessionid":"2b15e5c8-d3b1-11e4-b9d6-1681e6b88ec1","domain_sessionidx":3,"domain_userid":"bc2e92ec6c204a14","dvce_created_tstamp":"2013-11-26T00:03:57.885Z","etl_tstamp":"2013-11-26T00:03:57.885Z","event":"page_view","event_fingerprint":"e3dbfa9cca0412c3d4052863cefb547f","event_format":"jsonschema","event_id":"c6ef3124-b53a-4b13-a233-0088f79dcbcb","event_name":"link_click","event_vendor":"com.snowplowanalytics.snowplow","event_version":"1-0-0","geo_city":"New York","geo_country":"US","geo_latitude":37.443604,"geo_longitude":-122.4124,"geo_region":"TX","geo_region_name":"Florida","geo_zipcode":"94109","ip_domain":"nuvox.net","ip_isp":"FDN Communications","ip_netspeed":"Cable/DSL","ip_organization":"Bouygues Telecom","name_tracker":"cloudfront-1","network_userid":"ecdff4d0-9175-40ac-a8bb-325c49733607","page_title":"On Analytics","page_url":"http://www.snowplowanalytics.com","page_urlfragment":"4-conclusion","page_urlhost":"www.snowplowanalytics.com","page_urlpath":"/product/index.html","page_urlport":80,"page_urlquery":"id=GTM-DLRG","page_urlscheme":"http","platform":"web","true_tstamp":"2013-11-26T00:03:57.885Z","txn_id":41828,"unstruct_event_com_snowplowanalytics_snowplow_link_click_1":{"elementClasses":["foreground"],"elementId":"exampleLink","targetUrl":"http://www.example.com","unicodeTest":"<>angry_birds"},"user_fingerprint":"2161814971","user_id":"[email protected]","user_ipaddress":"92.231.54.234","v_collector":"clj-tomcat-0.1.0","v_etl":"serde-0.5.2","v_tracker":"js-2.1.0"}`)
+
 var eventMapWithGeo = map[string]interface{}{
 	"app_id":            "<>angry-birds",
 	"br_features_flash": false,
@@ -262,6 +266,7 @@ var eventMapWithGeo = map[string]interface{}{
 		"elementClasses": []interface{}{"foreground"},
 		"elementId":      "exampleLink",
 		"targetUrl":      "http://www.example.com",
+		"unicodeTest":    "<>angry_birds",
 	},
 	"user_fingerprint": "2161814971",
 	"user_id":          "[email protected]",
@@ -302,4 +307,4 @@ var subsetMap = map[string]interface{}{
 	"contexts_com_snowplowanalytics_snowplow_ua_parser_context_1": eventMapWithGeo["contexts_com_snowplowanalytics_snowplow_ua_parser_context_1"],
 }
 
-var subsetJson, _ = stdJson.Marshal(subsetMap)
+var subsetJson, _ = jsoniter.Marshal(subsetMap)