fixups

Author: mudler

backend/python/petals/backend_petals.py

@@ -75,6 +75,12 @@ def Predict(self, request, context):
         inputs = self.tokenizer(request.Prompt, return_tensors="pt")["input_ids"]
         if self.cuda:
             inputs = inputs.cuda()
+ 
+        if request.Tokens == 0:
+            # Max to max value if tokens are not specified
+            request.Tokens = 8192
+
+        # TODO: kwargs and map all parameters
         outputs = self.model.generate(inputs, max_new_tokens=request.Tokens)
 
         generated_text = self.tokenizer.decode(outputs[0])

backend/python/petals/run.sh

@@ -6,7 +6,14 @@
 export PATH=$PATH:/opt/conda/bin
 
 # Activate conda environment
-source activate petals
+# if source is available use it, or use conda
+#
+if [ -f /opt/conda/bin/activate ]; then
+    source activate petals
+else
+    eval "$(conda shell.bash hook)"
+    conda activate petals
+fi
 
 # get the directory where the bash script is located
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"