Dispose implicitly converted TorchSharp.Scalar and torch.Tensor

src/Examples.Utils/Examples.Utils.csproj

@@ -17,7 +17,7 @@
 
   <ItemGroup>
     <PackageReference Include="SharpZipLib" Version="1.4.0" />
-    <PackageReference Include="SixLabors.ImageSharp" Version="3.1.7" />
+    <PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
   </ItemGroup>
 
   <ItemGroup>

src/Examples/AdversarialExampleGeneration.cs

@@ -105,7 +105,9 @@ internal static void Main(string[] args)
         private static Tensor Attack(Tensor image, double ε, Tensor data_grad)
         {
             using (var sign = data_grad.sign()) {
-                var perturbed = (image + ε * sign).clamp(0.0, 1.0);
+                using var zero_scalar = 0.0.ToScalar();
+                using var one_scalar = 1.0.ToScalar();
+                var perturbed = (image + ε * sign).clamp(zero_scalar, one_scalar);
                 return perturbed;
             }
         }

src/Examples/SequenceToSequence.cs

@@ -234,10 +234,14 @@ public TransformerModel(long ntokens, long ninputs, long nheads, long nhidden, l
 
             public Tensor GenerateSquareSubsequentMask(long size)
             {
-                var mask = (torch.ones(new long[] { size, size }) == 1).triu().transpose(0, 1);
+                using var zero_scalar = 0.ToScalar();
+                using var one_scalar = 1.ToScalar();
+                using var float_negative_infinity_scalar = float.NegativeInfinity.ToScalar();
+                using var float_zero_scalar = 0.0f.ToScalar(); // FIXME: Equivalent to zero_scalar?
+                var mask = (torch.ones(new long[] { size, size }) == one_scalar).triu().transpose(0, 1);
                 return mask.to_type(ScalarType.Float32)
-                    .masked_fill(mask == 0, float.NegativeInfinity)
-                    .masked_fill(mask == 1, 0.0f).to(device);
+                    .masked_fill(mask == zero_scalar, float_negative_infinity_scalar)
+                    .masked_fill(mask == one_scalar, float_zero_scalar).to(device);
             }
 
             private void InitWeights()

src/Native/LibTorchSharp/THSTensor.h

@@ -1220,6 +1220,10 @@ EXPORT_API(Tensor) THSTensor_sqrt(const Tensor tensor);
 
 EXPORT_API(void) THSTensor_sqrt_(const Tensor tensor);
 
+EXPORT_API(Tensor) THSTensor_square(const Tensor tensor);
+
+EXPORT_API(void) THSTensor_square_(const Tensor tensor);
+
 EXPORT_API(Tensor) THSTensor_std(const Tensor tensor, const bool unbiased);
 
 EXPORT_API(Tensor) THSTensor_std_along_dimensions(const Tensor tensor, const int64_t* dimensions, int length, bool unbiased, bool keepdim);

src/Native/LibTorchSharp/THSTensorMath.cpp

@@ -910,6 +910,16 @@ void THSTensor_sqrt_(const Tensor tensor)
     CATCH(tensor->sqrt_();)
 }
 
+Tensor THSTensor_square(const Tensor tensor)
+{
+    CATCH_TENSOR(tensor->square());
+}
+
+void THSTensor_square_(const Tensor tensor)
+{
+    CATCH(tensor->square_();)
+}
+
 Tensor THSTensor_sign(const Tensor tensor)
 {
     CATCH_TENSOR(tensor->sign());

src/TorchAudio/Functional.cs

@@ -73,14 +73,15 @@ public static torch.Tensor spectrogram(torch.Tensor waveform, long pad, torch.Te
                     spec_f = spec_f.reshape(spec_shape);
 
                     if (normalized) {
-                        spec_f /= window.pow(2.0).sum().sqrt();
+                        spec_f /= window.square().sum().sqrt();
                     }
 
                     if (power.HasValue) {
                         if (power.Value == 1.0) {
                             spec_f = spec_f.abs();
                         } else {
-                            spec_f = spec_f.abs().pow(power.Value);
+                            using var power_scalar = power.Value.ToScalar();
+                            spec_f = spec_f.abs().pow(power_scalar); // FIXME: Call torch.Tensor.square if power.Value == 2.0?
                         }
                     }
 
@@ -112,7 +113,7 @@ public static torch.Tensor inverse_spectrogram(torch.Tensor spectrogram, long? l
                 using (var d = torch.NewDisposeScope()) {
 
                     if (normalized) {
-                        spectrogram = spectrogram * window.pow(2.0).sum().sqrt();
+                        spectrogram = spectrogram * window.square().sum().sqrt();
                     }
 
                     // pack batch
@@ -180,23 +181,24 @@ public static Tensor griffinlim(Tensor specgram, Tensor window, long n_fft, long
                     throw new ArgumentOutOfRangeException($"momentum must be in range [0, 1). Found: {momentum}");
                 }
                 momentum = momentum / (1 + momentum);
+                var need_momentum = momentum > 0.0;
+                using var momentum_scalar = (need_momentum) ? momentum.ToScalar() : null;
 
                 // pack batch
                 var shape = specgram.size();
                 specgram = specgram.reshape(new long[] { -1, shape[shape.Length - 2], shape[shape.Length - 1] });
 
-                specgram = specgram.pow(1 / power);
+                using var exponent_scalar = (1 / power).ToScalar();
+                specgram = specgram.pow(exponent_scalar); // FIXME: Use inplace ops? Skip if power == 1?
 
                 // initialize the phase
-                Tensor angles;
-                if (rand_init) {
-                    angles = torch.rand(specgram.size(), dtype: _get_complex_dtype(specgram.dtype), device: specgram.device);
-                } else {
-                    angles = torch.full(specgram.size(), 1, dtype: _get_complex_dtype(specgram.dtype), device: specgram.device);
-                }
+                var angles = (rand_init)
+                    ? torch.rand(specgram.size(), dtype: _get_complex_dtype(specgram.dtype), device: specgram.device)
+                    : torch.ones(specgram.size(), dtype: _get_complex_dtype(specgram.dtype), device: specgram.device);
 
                 // And initialize the previous iterate to 0
                 var tprev = torch.tensor(0.0, dtype: specgram.dtype, device: specgram.device);
+                using var eps_scalar = (1e-16).ToScalar();
                 for (int i = 0; i < n_iter; i++) {
                     // Invert with our current estimate of the phases
                     var inverse = torch.istft(
@@ -218,10 +220,10 @@ public static Tensor griffinlim(Tensor specgram, Tensor window, long n_fft, long
 
                     // Update our phase estimates
                     angles = rebuilt;
-                    if (momentum > 0.0) {
-                        angles = angles - tprev.mul_(momentum);
+                    if (need_momentum) {
+                        angles = angles - tprev.mul_(momentum_scalar!); // FIXME: Use inplace ops?
                     }
-                    angles = angles.div(angles.abs().add(1e-16));
+                    angles = angles.div(angles.abs().add(eps_scalar));
 
                     // Store the previous iterate
                     tprev = rebuilt;
@@ -528,18 +530,23 @@ internal static (torch.Tensor, int) _get_sinc_resample_kernel(int orig_freq, int
                 if (lowpass_filter_width <= 0) {
                     throw new ArgumentOutOfRangeException();
                 }
+                using var min_scalar = (-lowpass_filter_width).ToScalar();
+                using var max_scalar = lowpass_filter_width.ToScalar();
 
                 var kernels_list = new List<torch.Tensor>();
                 double base_freq = Math.Min(orig_freq, new_freq);
                 base_freq *= rolloff;
 
                 var width = (int)Math.Ceiling(((double)lowpass_filter_width) * orig_freq / base_freq);
                 var idx_dtype = dtype ?? torch.float64;
-                var idx = torch.arange(-width, width + orig_freq, device: device, dtype: idx_dtype);
+                using var start_scalar = (-width).ToScalar();
+                using var stop_scalar = (width + orig_freq).ToScalar();
+                var idx = torch.arange(start_scalar, stop_scalar, device: device, dtype: idx_dtype);
 
+                using var zero_scalar = 0.ToScalar();
                 for (int i = 0; i < new_freq; i++) {
                     var t = (-i / new_freq + idx / orig_freq) * base_freq;
-                    t = t.clamp_(-lowpass_filter_width, lowpass_filter_width);
+                    t = t.clamp_(min_scalar, max_scalar);
 
                     torch.Tensor window;
                     if (resampling_method == ResamplingMethod.sinc_interpolation) {
@@ -554,13 +561,14 @@ internal static (torch.Tensor, int) _get_sinc_resample_kernel(int orig_freq, int
                     }
                     t *= Math.PI;
                     // Tensor.to(Tensor) of TorchSharp desn't change dtype.
-                    var kernel = torch.where(t == 0, torch.tensor(1.0).to(t).type_as(t), torch.sin(t) / t);
+                    var kernel = torch.where(t == zero_scalar, torch.tensor(1.0).to(t).type_as(t), torch.sin(t) / t);
                     kernel.mul_(window);
                     kernels_list.Add(kernel);
                 }
 
                 var scale = ((double)base_freq) / orig_freq;
-                var kernels = torch.stack(kernels_list.ToArray()).view(new_freq, 1, -1).mul_(scale);
+                using var scale_scalar = scale.ToScalar();
+                var kernels = torch.stack(kernels_list.ToArray()).view(new_freq, 1, -1).mul_(scale_scalar);
                 if (dtype == null) {
                     kernels = kernels.to(torch.float32);
                 }

src/TorchAudio/Modules/HuBERTPretrainModel.cs

@@ -91,7 +91,7 @@ public override (Tensor?, Tensor?, Tensor) forward(
             if (this.feature_grad_mult != null && this.feature_grad_mult < 1.0) {
                 x = Wav2Vec2Model.GradMultiply.apply(x, this.feature_grad_mult.Value);
             }
-            var features_pen = x.@float().pow(2).mean();
+            var features_pen = x.@float().square().mean();
             if (lengths is not null) {
                 padding_mask = Wav2Vec2Model._get_padding_mask(x, lengths);
             } else {

src/TorchAudio/Modules/Tacotron2.cs

@@ -147,9 +147,11 @@ public override (Tensor, Tensor, Tensor, Tensor) forward(
                 mask = mask.expand(this.n_mels, mask.size(0), mask.size(1));
                 mask = mask.permute(1, 0, 2);
 
-                mel_specgram = mel_specgram.masked_fill(mask, 0.0);
-                mel_specgram_postnet = mel_specgram_postnet.masked_fill(mask, 0.0);
-                gate_outputs = gate_outputs.masked_fill(mask[TensorIndex.Colon, 0, TensorIndex.Colon], 1e3);
+                using var zero_scalar = 0.0.ToScalar();
+                mel_specgram = mel_specgram.masked_fill(mask, zero_scalar);
+                mel_specgram_postnet = mel_specgram_postnet.masked_fill(mask, zero_scalar);
+                using var eps_scalar = 1e3.ToScalar();
+                gate_outputs = gate_outputs.masked_fill(mask[TensorIndex.Colon, 0, TensorIndex.Colon], eps_scalar);
             }
 
             return (mel_specgram, mel_specgram_postnet, gate_outputs, alignments);
@@ -334,7 +336,8 @@ public override (Tensor, Tensor) forward(
             {
                 var alignment = this._get_alignment_energies(attention_hidden_state, processed_memory, attention_weights_cat);
 
-                alignment = alignment.masked_fill(mask, this.score_mask_value);
+                using var score_mask_value_scalar = this.score_mask_value.ToScalar();
+                alignment = alignment.masked_fill(mask, score_mask_value_scalar);
 
                 var attention_weights = F.softmax(alignment, dim: 1);
                 var attention_context = torch.bmm(attention_weights.unsqueeze(1), memory);

src/TorchAudio/Transforms/InverseMelScale.cs

@@ -96,18 +96,19 @@ public override Tensor forward(Tensor melspec)
                     learningRate: 0.1, momentum: 0.9);
 
                 var loss = float.PositiveInfinity;
+                using var zero_scalar = 0.ToScalar();
                 for (long i = 0; i < this.max_iter; i++) {
                     using var d2 = torch.NewDisposeScope();
 
                     optim.zero_grad();
                     var diff = melspec - specgram.matmul(this.fb);
-                    var new_loss = diff.pow(2).sum(dim: -1).mean();
+                    var new_loss = diff.square().sum(dim: -1).mean();
                     // take sum over mel-frequency then average over other dimensions
                     // so that loss threshold is applied par unit timeframe
                     new_loss.backward();
                     optim.step();
                     using (torch.no_grad())
-                        specgram.set_(specgram.clamp(min: 0));
+                        specgram.set_(specgram.clamp(min: zero_scalar));
 
                     var new_loss_value = new_loss.item<float>();
                     if (new_loss_value < this.tolerance_loss || Math.Abs(loss - new_loss_value) < this.tolerance_change) {
@@ -117,7 +118,7 @@ public override Tensor forward(Tensor melspec)
                 }
 
                 specgram.requires_grad_(false);
-                var specgram_tensor = specgram.clamp(min: 0).transpose(-1, -2);
+                var specgram_tensor = specgram.clamp(min: zero_scalar).transpose(-1, -2);
 
                 // unpack batch
                 shape[shape.Length - 2] = freq;

src/TorchSharp/Distributions/Beta.cs

@@ -28,7 +28,7 @@ public override Tensor variance {
                 get {
                     using var _ = NewDisposeScope();
                     var total = concentration0 + concentration1;
-                    return (concentration1 * concentration0 / (total.pow(2) * (total + 1))).MoveToOuterDisposeScope();
+                    return (concentration1 * concentration0 / (total.square() * (total + 1))).MoveToOuterDisposeScope();
                 }
             }
 

src/TorchSharp/Distributions/Cauchy.cs

@@ -78,7 +78,7 @@ public override Tensor rsample(params long[] sample_shape)
             /// </summary>
             /// <param name="value"></param>
             public override Tensor log_prob(Tensor value) =>
-                WrappedTensorDisposeScope(() => -Math.Log(Math.PI) - scale.log() - (((value - loc) / scale).pow(2)).log1p());            
+                WrappedTensorDisposeScope(() => -Math.Log(Math.PI) - scale.log() - (((value - loc) / scale).square()).log1p());
 
             /// <summary>
             /// Returns entropy of distribution, batched over batch_shape.

src/TorchSharp/Distributions/Constraints.cs

@@ -137,8 +137,10 @@ public _OneHot() : base(true, 1) { }
 
                     public override Tensor check(Tensor value)
                     {
-                        var is_boolean = (value == 0) | (value == 1);
-                        var is_normalized = value.sum(-1).eq(1);
+                        using var zero_scalar = 0.ToScalar();
+                        using var one_scalar = 1.ToScalar();
+                        var is_boolean = (value == zero_scalar) | (value == one_scalar);
+                        var is_normalized = value.sum(-1).eq(one_scalar);
                         return is_boolean.all(-1) & is_normalized;
                     }
                 }
@@ -433,9 +435,9 @@ public _CorrCholesky() : base(false, 2) { }
 
                     public override Tensor check(Tensor value)
                     {
-                        var tol = torch.finfo(value.dtype).eps * value.size(-1) * 10;  // 10 is an adjustable fudge factor
+                        using var tol_scalar = (torch.finfo(value.dtype).eps * value.size(-1) * 10).ToScalar();  // 10 is an adjustable fudge factor
                         var row_norm = torch.linalg.norm(value.detach(), dims: new[] { -1L });
-                        var unit_row_norm = (row_norm - 1.0).abs().le(tol).all(dim: -1);
+                        var unit_row_norm = (row_norm - 1.0).abs().le(tol_scalar).all(dim: -1);
                         return lc.check(value) & unit_row_norm;
                     }
 
@@ -489,7 +491,8 @@ public override Tensor check(Tensor value)
                         var sym_check = base.check(value);
                         if (!sym_check.all().item<bool>())
                             return sym_check;
-                        return torch.linalg.eigvalsh(value).ge(0).all(-1);
+                        using var zero_scalar = 0.ToScalar();
+                        return torch.linalg.eigvalsh(value).ge(zero_scalar).all(-1);
                     }
                 }
 
@@ -503,7 +506,8 @@ public override Tensor check(Tensor value)
                         var sym_check = base.check(value);
                         if (!sym_check.all().item<bool>())
                             return sym_check;
-                        return torch.linalg.cholesky_ex(value).info.eq(0);
+                        using var zero_scalar = 0.ToScalar();
+                        return torch.linalg.cholesky_ex(value).info.eq(zero_scalar);
                     }
                 }
 

src/TorchSharp/Distributions/Dirichlet.cs

@@ -25,9 +25,11 @@ public override Tensor mode
             {
                 get {
                     using var _ = NewDisposeScope();
-                    var concentrationm1 = (concentration - 1).clamp(min: 0.0);
+                    using var zero_scalar = 0.0.ToScalar();
+                    var concentrationm1 = (concentration - 1).clamp(min: zero_scalar);
                     var mode = concentrationm1 / concentrationm1.sum(-1, true);
-                    var mask = (concentration < 1).all(dim: -1);
+                    using var one_scalar = 1.ToScalar();
+                    var mask = (concentration < one_scalar).all(dim: -1);
                     mode[mask] = torch.nn.functional.one_hot(mode[mask].argmax(dim: -1), concentrationm1.shape[concentrationm1.ndim-1]).to(mode);
                     return mode.MoveToOuterDisposeScope();
                 }
@@ -40,7 +42,7 @@ public override Tensor variance {
                 get {
                     using var _ = NewDisposeScope();
                     var con0 = concentration.sum(-1, true);
-                    return (concentration * (con0 - concentration) / (con0.pow(2) * (con0 + 1))).MoveToOuterDisposeScope();
+                    return (concentration * (con0 - concentration) / (con0.square() * (con0 + 1))).MoveToOuterDisposeScope();
                 }
             }
 

src/TorchSharp/Distributions/Distribution.cs

@@ -183,10 +183,16 @@ protected Tensor ProbsToLogits(Tensor probs, bool isBinary = false)
                 protected Tensor ClampProbs(Tensor probs)
                 {
                     var eps = torch.finfo(probs.dtype).eps;
-                    return probs.clamp(eps, 1 - eps);
+                    using var eps_scalar = eps.ToScalar();
+                    using var eps_bar_scalar = (1 - eps).ToScalar();
+                    return probs.clamp(eps_scalar, eps_bar_scalar);
                 }
 
-                protected Tensor ClampByZero(Tensor x) => (x.clamp_min(0) + x - x.clamp_max(0)) / 2;
+                protected Tensor ClampByZero(Tensor x)
+                {
+                    using var zero_scalar = 0.ToScalar();
+                    return (x.clamp_min(zero_scalar) + x - x.clamp_max(zero_scalar)) / 2;
+                }
 
                 protected torch.Generator generator;
                 bool disposedValue;

src/TorchSharp/Distributions/ExpRelaxedCategorical.cs

@@ -134,7 +134,9 @@ public override Tensor log_prob(Tensor value)
                 var logitsValue = broadcast_tensors(_logits, value);
                 var logits = logitsValue[0];
                 value = logitsValue[1];
-                var log_scale = (torch.full_like(_temperature, K).lgamma() - _temperature.log().mul(-(K - 1)));
+                using var K_scalar = K.ToScalar();
+                using var negative_Ksub1_scalar = (-(K - 1)).ToScalar();
+                var log_scale = torch.full_like(_temperature, K_scalar).lgamma() - _temperature.log().mul(negative_Ksub1_scalar); // FIXME: Use inplace ops?
                 var score = logits - value.mul(_temperature);
                 score = (score - score.logsumexp(dim: -1, keepdim: true)).sum(-1);
                 return (score + log_scale).MoveToOuterDisposeScope();

src/TorchSharp/Distributions/Exponential.cs

@@ -24,7 +24,7 @@ public class Exponential : torch.distributions.ExponentialFamily
             /// <summary>
             /// The variance of the distribution
             /// </summary>
-            public override Tensor variance => rate.pow(2);
+            public override Tensor variance => rate.square();
 
             /// <summary>
             /// The standard deviation of the distribution