关于Transeger中的文本初始化 about svtas HOT 4 CLOSED

def forward(self, labels, masks):
    b, temporal_len = masks.shape
    prompts, pad_masks = self.prompt_learner(labels, b, temporal_len, masks.device)
    # [N T U D]
    prompts = prompts.to(masks.device)
    # [N T U 1] -> [N*T U 1]
    pad_masks = pad_masks.reshape([-1] + list(pad_masks.shape[2:])).to(masks. Device)

def forward(self, last_clip_labels, batch_size, temporal_len, device):
    #! this judge
    if last_clip_labels is None:
        start_promot = self._tokenizer.tokenize("").to(device)
        start_promot_embedding = self.token_embedding(start_promot)
        prompts = start_promot_embedding[:, :1].expand(batch_size, temporal_len // self.sample_rate, self.max_len, -1)
    else:
        text_list = []
        for b in range(batch_size):
            if torch.any(last_clip_labels[b,:] == self.ignore_index):
                end_promot = self._tokenizer.tokenize("").to(device)
                end_promot_embedding = self.token_embedding(end_promot)
                embedding = end_promot_embedding[:, 1:2].expand(1, temporal_len // self.sample_rate, self.max_len, -1)
                text_list.append(embedding)
            else:
                embedding = self.convert_id_to_promot(last_clip_labels[b, ::self.sample_rate])
                text_list.append(embedding.unsqueeze(0))
        
        # [N T U D]
        prompts = torch.cat(text_list, dim=0)
    pad_masks = torch.where(prompts != 0., torch.ones_like(prompts), torch.zeros_like(prompts))[:, :, :, 0:1]
    return prompts, pad_masks