From c73af6679c84a3c862a82fad228824a4c8fc1fff Mon Sep 17 00:00:00 2001 From: Dan Zheng Date: Tue, 5 Jan 2021 20:48:55 -0500 Subject: [PATCH 1/2] NFC: fix warnings related to `VectorProtocol.VectorSpaceScalar`. `VectorProtocol.VectorSpaceScalar` was changed from an associated type to a hardcoded typealias for `Float`. Fix warnings related to this change. --- Sources/TensorFlow/Layers/Sequential.swift | 123 +++--------------- .../TensorFlow/Layers/Sequential.swift.gyb | 9 +- .../TensorFlow/Optimizers/MomentumBased.swift | 21 +-- Sources/TensorFlow/Optimizers/SGD.swift | 3 +- 4 files changed, 27 insertions(+), 129 deletions(-) diff --git a/Sources/TensorFlow/Layers/Sequential.swift b/Sources/TensorFlow/Layers/Sequential.swift index 3631ec405..cf71989bd 100644 --- a/Sources/TensorFlow/Layers/Sequential.swift +++ b/Sources/TensorFlow/Layers/Sequential.swift @@ -47,8 +47,7 @@ import _Differentiation /// ```` public struct Sequential: Module where - Layer1.Output == Layer2.Input, - Layer1.TangentVector.VectorSpaceScalar == Layer2.TangentVector.VectorSpaceScalar + Layer1.Output == Layer2.Input { public var layer1: Layer1 public var layer2: Layer2 @@ -78,42 +77,28 @@ extension Sequential: Layer where Layer1: Layer { /// A layer that sequentially composes 3 layers. public typealias Sequential3 = Sequential> where - L1.Output == L2.Input, L2.Output == L3.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar + L1.Output == L2.Input, L2.Output == L3.Input /// A layer that sequentially composes 4 layers. public typealias Sequential4 = Sequential< L1, Sequential> > where - L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar + L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input /// A layer that sequentially composes 5 layers. public typealias Sequential5 = Sequential< L1, Sequential>> > where - L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, L4.Output == L5.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar + L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, L4.Output == L5.Input /// A layer that sequentially composes 6 layers. public typealias Sequential6 = Sequential>>>> where L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, L4.Output == L5.Input, - L5.Output == L6.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar + L5.Output == L6.Input /// A layer that sequentially composes 7 layers. public typealias Sequential7< @@ -123,13 +108,7 @@ public typealias Sequential7< > where L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, L4.Output == L5.Input, - L5.Output == L6.Input, L6.Output == L7.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar, - L6.TangentVector.VectorSpaceScalar == L7.TangentVector.VectorSpaceScalar + L5.Output == L6.Input, L6.Output == L7.Input /// A layer that sequentially composes 8 layers. public typealias Sequential8< @@ -140,14 +119,7 @@ public typealias Sequential8< > where L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, L4.Output == L5.Input, - L5.Output == L6.Input, L6.Output == L7.Input, L7.Output == L8.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar, - L6.TangentVector.VectorSpaceScalar == L7.TangentVector.VectorSpaceScalar, - L7.TangentVector.VectorSpaceScalar == L8.TangentVector.VectorSpaceScalar + L5.Output == L6.Input, L6.Output == L7.Input, L7.Output == L8.Input /// A layer that sequentially composes 9 layers. public typealias Sequential9< @@ -163,15 +135,7 @@ public typealias Sequential9< > where L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, L4.Output == L5.Input, - L5.Output == L6.Input, L6.Output == L7.Input, L7.Output == L8.Input, L8.Output == L9.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar, - L6.TangentVector.VectorSpaceScalar == L7.TangentVector.VectorSpaceScalar, - L7.TangentVector.VectorSpaceScalar == L8.TangentVector.VectorSpaceScalar, - L8.TangentVector.VectorSpaceScalar == L9.TangentVector.VectorSpaceScalar + L5.Output == L6.Input, L6.Output == L7.Input, L7.Output == L8.Input, L8.Output == L9.Input /// A layer that sequentially composes 10 layers. public typealias Sequential10< @@ -192,16 +156,7 @@ public typealias Sequential10< where L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, L4.Output == L5.Input, L5.Output == L6.Input, L6.Output == L7.Input, L7.Output == L8.Input, L8.Output == L9.Input, - L9.Output == L10.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar, - L6.TangentVector.VectorSpaceScalar == L7.TangentVector.VectorSpaceScalar, - L7.TangentVector.VectorSpaceScalar == L8.TangentVector.VectorSpaceScalar, - L8.TangentVector.VectorSpaceScalar == L9.TangentVector.VectorSpaceScalar, - L9.TangentVector.VectorSpaceScalar == L10.TangentVector.VectorSpaceScalar + L9.Output == L10.Input @resultBuilder public struct LayerBuilder { @@ -218,9 +173,7 @@ public struct LayerBuilder { -> Sequential> where L1.Output == L2.Input, - L2.Output == L3.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar + L2.Output == L3.Input { Sequential(l1, Sequential(l2, l3)) } @@ -235,10 +188,7 @@ public struct LayerBuilder { where L1.Output == L2.Input, L2.Output == L3.Input, - L3.Output == L4.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar + L3.Output == L4.Input { Sequential(l1, Sequential(l2, Sequential(l3, l4))) } @@ -255,11 +205,7 @@ public struct LayerBuilder { L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, - L4.Output == L5.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar + L4.Output == L5.Input { Sequential(l1, Sequential(l2, Sequential(l3, Sequential(l4, l5)))) } @@ -278,12 +224,7 @@ public struct LayerBuilder { L2.Output == L3.Input, L3.Output == L4.Input, L4.Output == L5.Input, - L5.Output == L6.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar + L5.Output == L6.Input { Sequential(l1, Sequential(l2, Sequential(l3, Sequential(l4, Sequential(l5, l6))))) } @@ -306,13 +247,7 @@ public struct LayerBuilder { L3.Output == L4.Input, L4.Output == L5.Input, L5.Output == L6.Input, - L6.Output == L7.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar, - L6.TangentVector.VectorSpaceScalar == L7.TangentVector.VectorSpaceScalar + L6.Output == L7.Input { Sequential( l1, Sequential(l2, Sequential(l3, Sequential(l4, Sequential(l5, Sequential(l6, l7)))))) @@ -341,14 +276,7 @@ public struct LayerBuilder { L4.Output == L5.Input, L5.Output == L6.Input, L6.Output == L7.Input, - L7.Output == L8.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar, - L6.TangentVector.VectorSpaceScalar == L7.TangentVector.VectorSpaceScalar, - L7.TangentVector.VectorSpaceScalar == L8.TangentVector.VectorSpaceScalar + L7.Output == L8.Input { Sequential( l1, @@ -384,15 +312,7 @@ public struct LayerBuilder { L5.Output == L6.Input, L6.Output == L7.Input, L7.Output == L8.Input, - L8.Output == L9.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar, - L6.TangentVector.VectorSpaceScalar == L7.TangentVector.VectorSpaceScalar, - L7.TangentVector.VectorSpaceScalar == L8.TangentVector.VectorSpaceScalar, - L8.TangentVector.VectorSpaceScalar == L9.TangentVector.VectorSpaceScalar + L8.Output == L9.Input { Sequential( l1, @@ -438,16 +358,7 @@ public struct LayerBuilder { L6.Output == L7.Input, L7.Output == L8.Input, L8.Output == L9.Input, - L9.Output == L10.Input, - L1.TangentVector.VectorSpaceScalar == L2.TangentVector.VectorSpaceScalar, - L2.TangentVector.VectorSpaceScalar == L3.TangentVector.VectorSpaceScalar, - L3.TangentVector.VectorSpaceScalar == L4.TangentVector.VectorSpaceScalar, - L4.TangentVector.VectorSpaceScalar == L5.TangentVector.VectorSpaceScalar, - L5.TangentVector.VectorSpaceScalar == L6.TangentVector.VectorSpaceScalar, - L6.TangentVector.VectorSpaceScalar == L7.TangentVector.VectorSpaceScalar, - L7.TangentVector.VectorSpaceScalar == L8.TangentVector.VectorSpaceScalar, - L8.TangentVector.VectorSpaceScalar == L9.TangentVector.VectorSpaceScalar, - L9.TangentVector.VectorSpaceScalar == L10.TangentVector.VectorSpaceScalar + L9.Output == L10.Input { Sequential( l1, diff --git a/Sources/TensorFlow/Layers/Sequential.swift.gyb b/Sources/TensorFlow/Layers/Sequential.swift.gyb index b1bf45afe..ceec34e29 100644 --- a/Sources/TensorFlow/Layers/Sequential.swift.gyb +++ b/Sources/TensorFlow/Layers/Sequential.swift.gyb @@ -46,8 +46,7 @@ import _Differentiation /// } /// ```` public struct Sequential: Module - where Layer1.Output == Layer2.Input, - Layer1.TangentVector.VectorSpaceScalar == Layer2.TangentVector.VectorSpaceScalar { + where Layer1.Output == Layer2.Input { public var layer1: Layer1 public var layer2: Layer2 @@ -94,8 +93,7 @@ def sequential_type(arity): %for n in SEQUENTIAL_ARITY_RANGE: /// A layer that sequentially composes ${n} layers. public typealias Sequential${n}<${generic_parameters(n, ", ")}> = ${sequential_type(n)} - where ${", ".join(["L{}.Output == L{}.Input".format(i, i+1) for i in range(1, n)])}, - ${", ".join(["L{}.TangentVector.VectorSpaceScalar == L{}.TangentVector.VectorSpaceScalar".format(i, i+1) for i in range(1, n)])} + where ${", ".join(["L{}.Output == L{}.Input".format(i, i+1) for i in range(1, n)])} %end @@ -112,9 +110,6 @@ public struct LayerBuilder { >(${", ".join(["_ l{}: L{}".format(i, i) for i in range(1, n+1)])}) -> ${sequential_type(n)} where ${",\n ".join(["L{}.Output == L{}.Input".format(i, i+1) - for i in range(1, n)])}, - ${",\n ".join(["L{}.TangentVector.VectorSpaceScalar == ".format(i) + - "L{}.TangentVector.VectorSpaceScalar".format(i+1) for i in range(1, n)])} { ${"".join(["Sequential(l{}, ".format(i) for i in range(1, n)])}l${n}${"".join([")" for _ in range(1, n)])} diff --git a/Sources/TensorFlow/Optimizers/MomentumBased.swift b/Sources/TensorFlow/Optimizers/MomentumBased.swift index fd0e3d2c7..92da45bed 100644 --- a/Sources/TensorFlow/Optimizers/MomentumBased.swift +++ b/Sources/TensorFlow/Optimizers/MomentumBased.swift @@ -33,8 +33,7 @@ import Numerics public class RMSProp: Optimizer where Model.TangentVector: VectorProtocol & PointwiseMultiplicative - & ElementaryFunctions & KeyPathIterable, - Model.TangentVector.VectorSpaceScalar == Float + & ElementaryFunctions & KeyPathIterable { public typealias Model = Model /// The learning rate. @@ -108,8 +107,7 @@ where public class AdaGrad: Optimizer where Model.TangentVector: VectorProtocol & PointwiseMultiplicative - & ElementaryFunctions & KeyPathIterable, - Model.TangentVector.VectorSpaceScalar == Float + & ElementaryFunctions & KeyPathIterable { public typealias Model = Model /// The learning rate. @@ -169,8 +167,7 @@ where public class AdaDelta: Optimizer where Model.TangentVector: VectorProtocol & PointwiseMultiplicative - & ElementaryFunctions & KeyPathIterable, - Model.TangentVector.VectorSpaceScalar == Float + & ElementaryFunctions & KeyPathIterable { public typealias Model = Model /// The learning rate. @@ -319,8 +316,7 @@ where public class Adam: Optimizer where Model.TangentVector: VectorProtocol & PointwiseMultiplicative - & ElementaryFunctions & KeyPathIterable, - Model.TangentVector.VectorSpaceScalar == Float + & ElementaryFunctions & KeyPathIterable { public typealias Model = Model /// The learning rate. @@ -403,8 +399,7 @@ where public class AdaMax: Optimizer where Model.TangentVector: VectorProtocol & PointwiseMultiplicative & ElementaryFunctions - & KeyPathIterable, - Model.TangentVector.VectorSpaceScalar == Float + & KeyPathIterable { public typealias Model = Model /// The learning rate. @@ -489,8 +484,7 @@ where public class AMSGrad: Optimizer where Model.TangentVector: VectorProtocol & PointwiseMultiplicative & ElementaryFunctions - & KeyPathIterable, - Model.TangentVector.VectorSpaceScalar == Float + & KeyPathIterable { public typealias Model = Model /// The learning rate. @@ -582,8 +576,7 @@ where public class RAdam: Optimizer where Model.TangentVector: VectorProtocol & PointwiseMultiplicative & ElementaryFunctions - & KeyPathIterable, - Model.TangentVector.VectorSpaceScalar == Float + & KeyPathIterable { public typealias Model = Model /// The learning rate. diff --git a/Sources/TensorFlow/Optimizers/SGD.swift b/Sources/TensorFlow/Optimizers/SGD.swift index 11dcf1eb5..ecf4e9a4e 100644 --- a/Sources/TensorFlow/Optimizers/SGD.swift +++ b/Sources/TensorFlow/Optimizers/SGD.swift @@ -38,8 +38,7 @@ import Numerics /// (Nesterov, 1983) public class SGD: Optimizer where - Model.TangentVector: VectorProtocol & ElementaryFunctions & KeyPathIterable, - Model.TangentVector.VectorSpaceScalar == Float + Model.TangentVector: VectorProtocol & ElementaryFunctions & KeyPathIterable { public typealias Model = Model /// The learning rate. From 34fca6a40f3ed0db948a0c3e005c72b6fb636838 Mon Sep 17 00:00:00 2001 From: Dan Zheng Date: Wed, 6 Jan 2021 03:27:14 -0500 Subject: [PATCH 2/2] Add explicit `Sequential.TangentVector: VectorProtocol` conformance. Fix type-checking error stating that `Sequential.TangentVector` does not satisfy `VectorProtocol.VectorSpaceScalar` requirement. --- Sources/TensorFlow/Layers/Sequential.swift | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Sources/TensorFlow/Layers/Sequential.swift b/Sources/TensorFlow/Layers/Sequential.swift index cf71989bd..30c4e3459 100644 --- a/Sources/TensorFlow/Layers/Sequential.swift +++ b/Sources/TensorFlow/Layers/Sequential.swift @@ -74,6 +74,13 @@ extension Sequential: Layer where Layer1: Layer { } } +extension Sequential.TangentVector: VectorProtocol { + /// - Note: this typealias declaration is necessary for building with old non-stock toolchains + /// where `VectorProtocol.VectorSpaceScalar` is an associated type and not a typealias + /// declaration. + typealias VectorSpaceScalar = Float +} + /// A layer that sequentially composes 3 layers. public typealias Sequential3 = Sequential> where