grad property

RELEASENOTES.md

@@ -8,6 +8,8 @@ __Breaking Changes__:
 
 __API Changes__:
 
+- #1291 `Tensor.grad()` and `Tensor.set_grad()` have been replaced by a new property `Tensor.grad`.
+
 __Bug Fixes__:
 
 

src/Examples/AdversarialExampleGeneration.cs

@@ -133,7 +133,7 @@ private static double Test(
                         model.zero_grad();
                         loss.backward();
 
-                        var perturbed = Attack(data, ε, data.grad());
+                        var perturbed = Attack(data, ε, data.grad);
 
                         using (var final = model.call(perturbed)) {
 

src/FSharp.Examples/AdversarialExampleGeneration.fs

@@ -79,7 +79,7 @@ let test (model:MNIST.Model) (eps:float) (data:Dataset) size =
             model.zero_grad()
             loss.backward()
 
-            use perturbed = attack input (eps.ToScalar()) (input.grad())
+            use perturbed = attack input (eps.ToScalar()) (input.grad)
             use final = perturbed --> model
             correct <- correct + final.argmax(1L).eq(labels).sum().ToInt32()
         end

src/TorchSharp/NN/Module.cs

@@ -240,7 +240,7 @@ private void _toEpilog(ScalarType? dtype, Device device)
 
                     foreach (var (name, param) in named_parameters(false).ToList()) {
                         if (!param.toWillCopy(dtype ?? param.dtype, device ?? param.device) &&
-                            (param.grad() is null || !param.grad().toWillCopy(dtype ?? param.dtype, device ?? param.device)))
+                            (param.grad is null || !param.grad.toWillCopy(dtype ?? param.dtype, device ?? param.device)))
                             continue;
 
                         Parameter p;
@@ -256,11 +256,10 @@ private void _toEpilog(ScalarType? dtype, Device device)
                                 .DetachFromDisposeScope() as Parameter;
 
                             // Copy the gradient over as well, if it exists
-                            var grad = param.grad();
+                            var grad = param.grad;
                             if (grad is not null) {
-                                p.set_grad(grad.to(paramType, device ?? param.device)
-                                                .with_requires_grad(grad.requires_grad)
-                                                .MoveToOtherDisposeScope(p));
+                                p.grad = grad.to(paramType, device ?? param.device)
+                                    .with_requires_grad(grad.requires_grad);
                             }
 
                             // Dispose the param and gradient
@@ -360,10 +359,10 @@ public virtual void zero_grad(bool set_to_none = true)
                     CheckForErrors();
 
                     foreach (var (_, p) in named_parameters()) {
-                        var grad = p.grad();
+                        var grad = p.grad;
                         if (grad is not null) {
                             if (set_to_none) {
-                                p.set_grad(null);
+                                p.grad = null;
                                 grad.DetachFromDisposeScope().Dispose();
                             } else {
                                 grad.zero_();

src/TorchSharp/Optimizers/ASGD.cs

@@ -145,7 +145,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/Adadelta.cs

@@ -136,7 +136,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = (maximize) ? -param.grad() : param.grad();
+                        var grad = (maximize) ? -param.grad : param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/Adagrad.cs

@@ -147,7 +147,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                         var state = (State)_state[param.handle];
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/Adam.cs

@@ -164,7 +164,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                         var state = (State)_state[param.handle];
 
-                        var grad = (maximize) ? -param.grad() : param.grad();
+                        var grad = (maximize) ? -param.grad : param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/AdamW.cs

@@ -164,7 +164,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                         var state = (State)_state[param.handle];
 
-                        var grad = (maximize) ? -param.grad() : param.grad();
+                        var grad = (maximize) ? -param.grad : param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/Adamax.cs

@@ -148,7 +148,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/NAdam.cs

@@ -154,7 +154,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/Optimizer.cs

@@ -399,7 +399,7 @@ public override void zero_grad()
 
                     foreach (var p in g.Parameters) {
 
-                        using var grad = p.grad();
+                        using var grad = p.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/RAdam.cs

@@ -147,7 +147,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/RMSprop.cs

@@ -162,7 +162,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                         var state = (State)_state[param.handle];
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/Rprop.cs

@@ -144,7 +144,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                     foreach (var param in group.Parameters) {
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Optimizers/SGD.cs

@@ -150,7 +150,7 @@ public override Tensor step(Func<Tensor> closure = null)
 
                         var state = (State)_state[param.handle];
 
-                        var grad = param.grad();
+                        var grad = param.grad;
 
                         if (grad is null) continue;
 

src/TorchSharp/Tensor/Tensor.cs

@@ -1340,25 +1340,21 @@ public Tensor pin_memory()
             /// This attribute is null by default and becomes a Tensor the first time a call to backward() computes gradients for the tensor.
             /// The attribute will then contain the gradients computed and future calls to backward() will accumulate (add) gradients into it.
             /// </summary>
-            public Tensor? grad()
-            {
-                var res = NativeMethods.THSTensor_grad(Handle);
-                CheckForErrors();
-
-                if (res == IntPtr.Zero)
-                    return null;
+            public Tensor? grad {
+                get {
+                    var res = NativeMethods.THSTensor_grad(Handle);
+                    CheckForErrors();
 
-                return new Tensor(res);
-            }
+                    if (res == IntPtr.Zero)
+                        return null;
 
-            /// <summary>
-            /// This function will set the `tensor.grad()` attribute to a custom tensor. 
-            /// </summary>
-            /// <param name="grad">The new gradient tensor</param>
-            public void set_grad(Tensor grad)
-            {
-                NativeMethods.THSTensor_set_grad(Handle, grad?.DetachFromDisposeScope().Handle ?? IntPtr.Zero);
-                CheckForErrors();
+                    return new Tensor(res);
+                }
+                set {
+                    value?.DetachFromDisposeScope();
+                    NativeMethods.THSTensor_set_grad(Handle, value?.Handle ?? IntPtr.Zero);
+                    CheckForErrors();
+                }
             }
 
             internal void EncodeIndices(TensorIndex[] indices,

test/TorchSharpTest/NN.cs

@@ -2152,14 +2152,14 @@ public void TestBackward()
             output.backward();
 
             foreach (var parm in seq.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.NotNull(grad);
             }
 
             seq.zero_grad();
 
             foreach (var parm in seq.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.True(grad is null || grad!.count_nonzero().item<long>() == 0);
             }
         }
@@ -2186,14 +2186,14 @@ public void TestGettingParameters()
             output.backward();
 
             foreach (var parm in seq.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.NotNull(grad);
             }
 
             seq.zero_grad();
 
             foreach (var parm in seq.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.True(grad is null || grad!.count_nonzero().item<long>() == 0);
             }
         }
@@ -2220,14 +2220,14 @@ public void TestGrad()
             output.backward();
 
             foreach (var parm in seq.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.NotNull(grad);
             }
 
             seq.zero_grad();
 
             foreach (var parm in seq.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.True(grad is null || grad!.count_nonzero().item<long>() == 0);
             }
         }
@@ -2254,9 +2254,9 @@ public void TestGrad2()
 
             output.backward();
 
-            var scalerGrad = scaler.grad();
-            var weightGrad = linear.weight.grad();
-            var biasGrad = linear.bias.grad();
+            var scalerGrad = scaler.grad;
+            var weightGrad = linear.weight.grad;
+            var biasGrad = linear.bias.grad;
             Assert.True(scalerGrad is not null && scalerGrad.shape.Length == 2);
             Assert.True(weightGrad is not null && weightGrad.shape.Length == 2);
             Assert.True(biasGrad is not null && biasGrad.shape.Length == 2);
@@ -2328,7 +2328,7 @@ public void TestGradConditional()
             var gradCounts = 0;
 
             foreach (var (name, parm) in modT.named_parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 gradCounts += grad is not null ? (grad.Handle == IntPtr.Zero ? 0 : 1) : 0;
             }
 
@@ -2346,7 +2346,7 @@ public void TestGradConditional()
             gradCounts = 0;
 
             foreach (var parm in modF.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 gradCounts += grad is not null ? (grad.Handle == IntPtr.Zero ? 0 : 1) : 0;
             }
 
@@ -2839,14 +2839,14 @@ public void TestCustomModule1()
             output.backward();
 
             foreach (var (pName, parm) in module.named_parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.NotNull(grad);
             }
 
             module.zero_grad();
 
             foreach (var (pName, parm) in module.named_parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.True(grad is null || grad!.count_nonzero().item<long>() == 0);
             }
 
@@ -3016,7 +3016,7 @@ public void TestDerivedSequence1Grad()
             output.backward();
 
             foreach (var parm in seq.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
             }
         }
 
@@ -3037,7 +3037,7 @@ public void TestDerivedSequence2Grad()
             output.backward();
 
             foreach (var parm in seq.parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
             }
         }
 
@@ -3121,7 +3121,7 @@ public void TestCustomModuleWithDeviceMove()
                 var y = torch.randn(2, device: torch.CUDA);
                 torch.nn.functional.mse_loss(module.call(x), y).backward();
                 foreach (var (pName, parm) in module.named_parameters()) {
-                    var grad = parm.grad();
+                    var grad = parm.grad;
                     Assert.NotNull(grad);
                 }
 
@@ -3134,7 +3134,7 @@ public void TestCustomModuleWithDeviceMove()
                 y = torch.randn(2);
                 torch.nn.functional.mse_loss(module.call(x), y).backward();
                 foreach (var (pName, parm) in module.named_parameters()) {
-                    var grad = parm.grad();
+                    var grad = parm.grad;
                     Assert.NotNull(grad);
                 }
             }
@@ -3151,7 +3151,7 @@ public void TestCustomModuleWithTypeMove()
             var y = torch.randn(2, float64);
             torch.nn.functional.mse_loss(module.call(x), y).backward();
             foreach (var (pName, parm) in module.named_parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.NotNull(grad);
             }
 
@@ -3164,7 +3164,7 @@ public void TestCustomModuleWithTypeMove()
             y = torch.randn(2);
             torch.nn.functional.mse_loss(module.call(x), y).backward();
             foreach (var (pName, parm) in module.named_parameters()) {
-                var grad = parm.grad();
+                var grad = parm.grad;
                 Assert.NotNull(grad);
             }
         }
@@ -3180,7 +3180,7 @@ public void TestCustomModuleWithDeviceAndTypeMove()
                 var y = torch.randn(2, float16, torch.CUDA);
                 torch.nn.functional.mse_loss(module.call(x), y).backward();
                 foreach (var (pName, parm) in module.named_parameters()) {
-                    var grad = parm.grad();
+                    var grad = parm.grad;
                     Assert.NotNull(grad);
                 }
 
@@ -3193,7 +3193,7 @@ public void TestCustomModuleWithDeviceAndTypeMove()
                 y = torch.randn(2);
                 torch.nn.functional.mse_loss(module.call(x), y).backward();
                 foreach (var (pName, parm) in module.named_parameters()) {
-                    var grad = parm.grad();
+                    var grad = parm.grad;
                     Assert.NotNull(grad);
                 }
             }