Merge branch 'release/0.11.0'

Author: ianoc

.travis.yml

@@ -2,9 +2,9 @@ language: scala
 sudo: false
 matrix:
   include:
-    - scala: 2.10.4
+    - scala: 2.10.5
       script: ./sbt ++$TRAVIS_SCALA_VERSION clean test
 
-    - scala: 2.11.5
+    - scala: 2.11.7
       script: ./sbt ++$TRAVIS_SCALA_VERSION clean test
       after_success: "./sbt coveralls"

CHANGES.md

@@ -1,6 +1,29 @@
 # Algebird #
 
+### Version 0.11.0 ###
+* Move CMSHasherByteArray from scalding: https://github.com/twitter/algebird/pull/467
+* Upgrade sbt launcher script (sbt-extras): https://github.com/twitter/algebird/pull/469
+* Create case class macros for algebraic structures: https://github.com/twitter/algebird/pull/466
+* Refactor MapAggregator: https://github.com/twitter/algebird/pull/462
+* Algebird support for spark: https://github.com/twitter/algebird/pull/397
+* Add MapAggregator from 1 (key, aggregator) pair: https://github.com/twitter/algebird/pull/452
+* Remove unnecessary use of scala.math: https://github.com/twitter/algebird/pull/455
+* Don't call deprecated HyperLogLog methods in tests: https://github.com/twitter/algebird/pull/456
+* Update product_generators.rb: https://github.com/twitter/algebird/pull/457
+* Pzheng/gaussian euclidean: https://github.com/twitter/algebird/pull/448
+
+### Version 0.10.2 ###
+* QTree quantileBounds assert percentile <= 1.0 #447
+
+### Version 0.10.1 ###
+* Make HLL easier to use, add Hash128 typeclass #440
+* add ! to ApproximateBoolean #442
+* add QTreeAggregator and add approximatePercentileBounds to Aggregator #443
+* Make level configurable in QTreeAggregators #444
+
 ### Version 0.10.0 ###
+* HyperLogLogSeries #295
+* CMS: add contramap to convert CMS[K] to CMS[L], add support for String and Bytes, remove Ordering context bound for K #399
 * EventuallyAggregator and variants #407
 * Add MultiAggregator.apply #408
 * Return a MonoidAggregator from MultiAggregator when possible #409

README.md

@@ -1,5 +1,5 @@
 ## Algebird [![Build status](https://img.shields.io/travis/twitter/algebird/develop.svg)](http://travis-ci.org/twitter/algebird) [![Coverage status](https://img.shields.io/coveralls/twitter/algebird/develop.svg)](https://coveralls.io/r/twitter/algebird?branch=develop)
- 		
+
 
 Abstract algebra for Scala. This code is targeted at building aggregation systems (via [Scalding](https://github.com/twitter/scalding) or [Storm](https://github.com/nathanmarz/storm)). It was originally developed as part of Scalding's Matrix API, where Matrices had values which are elements of Monoids, Groups, or Rings. Subsequently, it was clear that the code had broader application within Scalding and on other projects within Twitter.
 
@@ -10,7 +10,7 @@ See the [current API documentation](http://twitter.github.com/algebird) for more
 ```scala
 > ./sbt algebird-core/console
 
-Welcome to Scala version 2.9.3 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_07).
+Welcome to Scala version 2.10.5 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_40).
 Type in expressions to have them evaluated.
 Type :help for more information.
 
@@ -21,7 +21,7 @@ scala> import com.twitter.algebird.Operators._
 import com.twitter.algebird.Operators._
 
 scala> Map(1 -> Max(2)) + Map(1 -> Max(3)) + Map(2 -> Max(4))
-res1: scala.collection.immutable.Map[Int,com.twitter.algebird.Max[Int]] = Map(2 -> Max(4), 1 -> Max(3))
+res0: scala.collection.immutable.Map[Int,com.twitter.algebird.Max[Int]] = Map(2 -> Max(4), 1 -> Max(3))
 ```
 In the above, the class Max[T] signifies that the + operator should actually be max (this is
 accomplished by providing an implicit instance of a typeclass for Max that handles +).
@@ -48,7 +48,7 @@ Discussion occurs primarily on the [Algebird mailing list](https://groups.google
 
 ## Maven
 
-Algebird modules are available on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.10.0`.
+Algebird modules are available on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.11.0`.
 
 Current published artifacts are
 
@@ -91,6 +91,6 @@ The answer is a mix of the following:
 * Argyris Zymnis <http://twitter.com/argyris>
 
 ## License
-Copyright 2012 Twitter, Inc.
+Copyright 2015 Twitter, Inc.
 
 Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0

algebird-core/src/main/scala/com/twitter/algebird/Aggregator.scala

@@ -116,10 +116,34 @@ object Aggregator extends java.io.Serializable {
   /**
    * This builds an in-memory Set, and then finally gets the size of that set.
    * This may not be scalable if the Uniques are very large. You might check the
-   * HyperLogLog Aggregator to get an approximate version of this that is scalable.
+   * approximateUniqueCount or HyperLogLog Aggregator to get an approximate version
+   * of this that is scalable.
    */
   def uniqueCount[T]: MonoidAggregator[T, Set[T], Int] =
     toSet[T].andThenPresent(_.size)
+
+  /**
+   * Using a constant amount of memory, give an approximate unique count (~ 1% error).
+   * This uses an exact set for up to 100 items,
+   * then HyperLogLog (HLL) with an 1.2% standard error which uses at most 8192 bytes
+   * for each HLL. For more control, see HyperLogLogAggregator.
+   */
+  def approximateUniqueCount[T: Hash128]: MonoidAggregator[T, Either[HLL, Set[T]], Long] =
+    SetSizeHashAggregator[T](hllBits = 13, maxSetSize = 100)
+
+  /**
+   * Returns the lower bound of a given percentile where the percentile is between (0,1]
+   * The items that are iterated over cannot be negative.
+   */
+  def approximatePercentile[T](percentile: Double, k: Int)(implicit num: Numeric[T]): QTreeAggregatorLowerBound[T] =
+    QTreeAggregatorLowerBound[T](percentile, k)
+
+  /**
+   * Returns the intersection of a bounded percentile where the percentile is between (0,1]
+   * The items that are iterated over cannot be negative.
+   */
+  def approximatePercentileBounds[T](percentile: Double, k: Int)(implicit num: Numeric[T]): QTreeAggregator[T] =
+    QTreeAggregator[T](percentile, k)
 }
 
 /**

algebird-core/src/main/scala/com/twitter/algebird/Approximate.scala

@@ -22,6 +22,8 @@ case class ApproximateBoolean(isTrue: Boolean, withProb: Double) {
 
   def not: ApproximateBoolean = ApproximateBoolean(!isTrue, withProb)
 
+  def unary_! : ApproximateBoolean = not
+
   def ^(that: ApproximateBoolean): ApproximateBoolean = {
     // This is true with probability > withProb * that.withProb
     // The answer is also correct if both are wrong, which is

algebird-core/src/main/scala/com/twitter/algebird/CountMinSketch.scala

@@ -1207,4 +1207,8 @@ object CMSHasherImplicits {
     override def hash(a: Int, b: Int, width: Int)(x: Bytes): Int = hashBytes(a, b, width)(x.array)
   }
 
-}
+  implicit object CMSHasherByteArray extends CMSHasher[Array[Byte]] {
+    override def hash(a: Int, b: Int, width: Int)(x: Array[Byte]): Int = hashBytes(a, b, width)(x)
+  }
+
+}

algebird-core/src/main/scala/com/twitter/algebird/DecayedValue.scala

@@ -24,14 +24,14 @@ package com.twitter.algebird
 
 object DecayedValue extends java.io.Serializable {
   def build[V <% Double](value: V, time: Double, halfLife: Double) = {
-    DecayedValue(value, time * scala.math.log(2.0) / halfLife)
+    DecayedValue(value, time * math.log(2.0) / halfLife)
   }
   val zero = DecayedValue(0.0, Double.NegativeInfinity)
 
   def scale(newv: DecayedValue, oldv: DecayedValue, eps: Double) = {
     val newValue = newv.value +
-      scala.math.exp(oldv.scaledTime - newv.scaledTime) * oldv.value
-    if (scala.math.abs(newValue) > eps) {
+      math.exp(oldv.scaledTime - newv.scaledTime) * oldv.value
+    if (math.abs(newValue) > eps) {
       DecayedValue(newValue, newv.scaledTime)
     } else {
       zero