Use 2.19.1 DLAMI as source AMI (#688)

* feat(ami): use 2.19.1 DLAMI source image

* test(decoder): special tokens are added by default

* fix(ami): use new venv path

* fix(ami): workaround read-only .bashrc

infrastructure/ami/hcl2-files/build.pkr.hcl

@@ -14,7 +14,7 @@ build {
     ]
   }
   provisioner "shell" {
-    inline = ["echo 'source /opt/aws_neuron_venv_pytorch/bin/activate' >> /home/ubuntu/.bashrc"]
+    inline = ["echo 'source /opt/aws_neuronx_venv_pytorch_2_1/bin/activate' | sudo tee -a /home/ubuntu/.bashrc"]
   }
   provisioner "file" {
     source      = "scripts/welcome-msg.sh"
@@ -26,4 +26,4 @@ build {
       "sudo chmod +x /etc/update-motd.d/99-custom-message",
     ]
   }
-}
+}

infrastructure/ami/hcl2-files/variables.pkr.hcl

@@ -10,15 +10,15 @@ variable "instance_type" {
 }
 
 variable "source_ami" {
-  default     = "ami-0274e546d67626305"
+  default     = "ami-0bcb701dd3cace633"
   description = "Base Image"
   type        = string
   /*
   To get latest value, run the following command:
   aws ec2 describe-images \
       --region us-east-1 \
       --owners amazon \
-      --filters 'Name=name,Values=Deep Learning AMI Neuron PyTorch 1.13 (Ubuntu 20.04) ????????' 'Name=state,Values=available' \
+      --filters 'Name=name,Values=Deep Learning AMI Neuron ????????' 'Name=state,Values=available' \
       --query 'reverse(sort_by(Images, &CreationDate))[:1].ImageId' \
       --output text
   */

infrastructure/ami/scripts/install-huggingface-libraries.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Activate the neuron virtual environment
-source /opt/aws_neuron_venv_pytorch/bin/activate
+source /opt/aws_neuronx_venv_pytorch_2_1/bin/activate
 
 echo "Step: install-hugging-face-libraries"
 
@@ -34,4 +34,4 @@ rm -rf optimum-neuron
 chmod -R 777 /home/ubuntu/huggingface-neuron-samples /home/ubuntu/huggingface-neuron-notebooks
 
 echo "Step: validate-imports-of-huggingface-libraries"
-bash -c 'python -c "import transformers;import datasets;import accelerate;import evaluate;import tensorboard; import torch;from optimum.neuron import pipeline"'
+bash -c 'python -c "import transformers;import datasets;import accelerate;import evaluate;import tensorboard; import torch;from optimum.neuron import pipeline"'

infrastructure/ami/scripts/validate-neuron.sh

@@ -3,11 +3,11 @@ echo "Step: validate-neuron-devices"
 neuron-ls
 
 # Activate the neuron virtual environment
-source /opt/aws_neuron_venv_pytorch/bin/activate
+source /opt/aws_neuronx_venv_pytorch_2_1/bin/activate
 
 python -c 'import torch'
 python -c 'import torch_neuronx'
 
 echo "Installing Tensorboard Plugin for Neuron"
 pip install --upgrade --no-cache-dir \
-    "tensorboard-plugin-neuronx"
+    "tensorboard-plugin-neuronx"

tests/decoder/test_decoder_pipelines.py

@@ -35,7 +35,7 @@ def _test_generation(p):
             # We only ever generate one sequence per input
             sequence = output[0]
             if return_tensors:
-                input_ids = p.tokenizer(input, add_special_tokens=False).input_ids
+                input_ids = p.tokenizer(input).input_ids
                 assert sequence["generated_token_ids"][: len(input_ids)] == input_ids
             else:
                 assert sequence["generated_text"].startswith(input)