Upload sample-sigplan.tex

sample-sigplan.tex

@@ -0,0 +1,744 @@
+%%
+%% This is file `sample-sigplan.tex',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% samples.dtx  (with options: `sigplan')
+%% 
+%% IMPORTANT NOTICE:
+%% 
+%% For the copyright see the source file.
+%% 
+%% Any modified versions of this file must be renamed
+%% with new filenames distinct from sample-sigplan.tex.
+%% 
+%% For distribution of the original source see the terms
+%% for copying and modification in the file samples.dtx.
+%% 
+%% This generated file may be distributed as long as the
+%% original source files, as listed above, are part of the
+%% same distribution. (The sources need not necessarily be
+%% in the same archive or directory.)
+%%
+%%
+%% Commands for TeXCount
+%TC:macro \cite [option:text,text]
+%TC:macro \citep [option:text,text]
+%TC:macro \citet [option:text,text]
+%TC:envir table 0 1
+%TC:envir table* 0 1
+%TC:envir tabular [ignore] word
+%TC:envir displaymath 0 word
+%TC:envir math 0 word
+%TC:envir comment 0 0
+%%
+%%
+%% The first command in your LaTeX source must be the \documentclass
+%% command.
+%%
+%% For submission and review of your manuscript please change the
+%% command to \documentclass[manuscript, screen, review]{acmart}.
+%%
+%% When submitting camera ready or to TAPS, please change the command
+%% to \documentclass[sigconf]{acmart} or whichever template is required
+%% for your publication.
+%%
+%%
+\documentclass[sigplan,screen]{acmart}
+
+%%
+%% \BibTeX command to typeset BibTeX logo in the docs
+\AtBeginDocument{%
+  \providecommand\BibTeX{{%
+    Bib\TeX}}}
+
+%% Rights management information.  This information is sent to you
+%% when you complete the rights form.  These commands have SAMPLE
+%% values in them; it is your responsibility as an author to replace
+%% the commands and values with those provided to you when you
+%% complete the rights form.
+\setcopyright{acmcopyright}
+\copyrightyear{2023}
+\acmYear{2023}
+\acmDOI{XXXXXXX.XXXXXXX}
+
+%% These commands are for a PROCEEDINGS abstract or paper.
+\acmConference[MM '23]{Make sure to enter the correct
+  conference title from your rights confirmation emai}{October 29--November 03,
+  2023}{Ottawa, Canada}
+\acmPrice{15.00}
+\acmISBN{978-1-4503-XXXX-X/18/06}
+\usepackage{algorithm}  
+\usepackage{algorithmic}
+\usepackage{setspace}  
+\usepackage{color}
+\usepackage{subfigure}
+%%
+%% Submission ID.
+%% Use this when submitting an article to a sponsored event. You'll
+%% receive a unique submission ID from the organizers
+%% of the event, and this ID should be used as the parameter to this command.
+%%\acmSubmissionID{123-A56-BU3}
+
+%%
+%% For managing citations, it is recommended to use bibliography
+%% files in BibTeX format.
+%%
+%% You can then either use BibTeX with the ACM-Reference-Format style,
+%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
+%% support for advanced citation of software artefact from the
+%% biblatex-software package, also separately available on CTAN.
+%%
+%% Look at the sample-*-biblatex.tex files for templates showcasing
+%% the biblatex styles.
+%%
+
+%%
+%% The majority of ACM publications use numbered citations and
+%% references.  The command \citestyle{authoryear} switches to the
+%% "author year" style.
+%%
+%% If you are preparing content for an event
+%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
+%% citations and references.
+%% Uncommenting
+%% the next command will enable that style.
+%%\citestyle{acmauthoryear}
+
+
+%%
+%% end of the preamble, start of the body of the document source.
+\begin{document}
+
+%%
+%% The "title" command has an optional parameter,
+%% allowing the author to define a "short title" to be used in page headers.
+\title{Boosting Sentence Representation with Visually-supervised Multimodal Pre-training}
+
+%%
+%% The "author" command and its associated commands are used to define
+%% the authors and their affiliations.
+%% Of note is the shared affiliation of the first two authors, and the
+%% "authornote" and "authornotemark" commands
+%% used to denote shared contribution to the research.
+\author{Zhe Li}
+%\authornote{Both authors contributed equally to this research.}
+\email{keycharon0122@gmail.com}
+\orcid{1234-5678-9012}
+%\author{G.K.M. Tobin}
+%\authornotemark[1]
+%\email{webmaster@marysville-ohio.com}
+\affiliation{%
+  \institution{Huazhong University of Science and Technology}
+  \city{WuHan}
+  \country{China}
+  \postcode{430000}
+}
+
+%\author{Lars Th{\o}rv{\"a}ld}
+%\affiliation{%
+%  \institution{The Th{\o}rv{\"a}ld Group}
+%  \streetaddress{1 Th{\o}rv{\"a}ld Circle}
+%  \city{Hekla}
+%  \country{Iceland}}
+%\email{larst@affiliation.org}
+%
+%\author{Valerie B\'eranger}
+%\affiliation{%
+%  \institution{Inria Paris-Rocquencourt}
+%  \city{Rocquencourt}
+%  \country{France}
+%}
+%
+%\author{Aparna Patel}
+%\affiliation{%
+% \institution{Rajiv Gandhi University}
+% \streetaddress{Rono-Hills}
+% \city{Doimukh}
+% \state{Arunachal Pradesh}
+% \country{India}}
+%
+%\author{Huifen Chan}
+%\affiliation{%
+%  \institution{Tsinghua University}
+%  \streetaddress{30 Shuangqing Rd}
+%  \city{Haidian Qu}
+%  \state{Beijing Shi}
+%  \country{China}}
+%
+%\author{Charles Palmer}
+%\affiliation{%
+%  \institution{Palmer Research Laboratories}
+%  \streetaddress{8600 Datapoint Drive}
+%  \city{San Antonio}
+%  \state{Texas}
+%  \country{USA}
+%  \postcode{78229}}
+%\email{cpalmer@prl.com}
+%
+%\author{John Smith}
+%\affiliation{%
+%  \institution{The Th{\o}rv{\"a}ld Group}
+%  \streetaddress{1 Th{\o}rv{\"a}ld Circle}
+%  \city{Hekla}
+%  \country{Iceland}}
+%\email{jsmith@affiliation.org}
+%
+%\author{Julius P. Kumquat}
+%\affiliation{%
+%  \institution{The Kumquat Consortium}
+%  \city{New York}
+%  \country{USA}}
+%\email{jpkumquat@consortium.net}
+
+%%
+%% By default, the full list of authors will be used in the page
+%% headers. Often, this list is too long, and will overlap
+%% other information printed in the page headers. This command allows
+%% the author to define a more concise list
+%% of authors' names for this purpose.
+\renewcommand{\shortauthors}{Zhe Li et al.}
+
+%%
+%% The abstract is a short summary of the work to be presented in the
+%% article.
+\begin{abstract}
+Large-scale pre-trained language models have garnered significant attention owing to its applicability to extracting sentence representations. Most pre-trained models use encoder based on transformer with single modality, performing well in natural language inference and question-answering. However, multimodal data can provide more effective features from different modalities. Unfortunately, existing multimodal pre-trained models either lack modality alignment, or do not explore complementary information between modalities, even cannot distinguish highly similar negative and positive samples. To alleviate these issues, we propose a Visually-supervised Pre-trained Multimodal Model (ViPMM) for sentence representation. We design diverse label-free multimodal proxy tasks to embed visual information into language. Our comprehensive downstream experiments on natural language understanding and sentiment classification show that ViPMM outperforms both existing unimodal and multimodal pre-trained models.
+\end{abstract}
+
+\begin{CCSXML}
+	<ccs2012>
+	<concept>
+	<concept_id>10010147.10010178</concept_id>
+	<concept_desc>Computing methodologies~Artificial intelligence</concept_desc>
+	<concept_significance>500</concept_significance>
+	</concept>
+	<concept>
+	<concept_id>10010147.10010178.10010179</concept_id>
+	<concept_desc>Computing methodologies~Natural language processing</concept_desc>
+	<concept_significance>500</concept_significance>
+	</concept>
+	</ccs2012>
+\end{CCSXML}
+
+\ccsdesc[500]{Computing methodologies~Artificial intelligence}
+\ccsdesc[500]{Computing methodologies~Natural language processing}
+%%
+%% Keywords. The author(s) should pick words that accurately describe
+%% the work being presented. Separate the keywords with commas.
+\keywords{multimodal pre-trained, visually-supervised, sentence representation, proxy task}
+%% A "teaser" image appears between the author and affiliation
+%% information and the body of the document, and typically spans the
+%% page.
+%\begin{teaserfigure}
+%  \includegraphics[width=\textwidth]{sampleteaser}
+%  \caption{Seattle Mariners at Spring Training, 2010.}
+%  \Description{Enjoying the baseball game from the third-base
+%  seats. Ichiro Suzuki preparing to bat.}
+%  \label{fig:teaser}
+%\end{teaserfigure}
+
+%\received{20 February 2007}
+%\received[revised]{12 March 2009}
+%\received[accepted]{5 June 2009}
+
+%%
+%% This command processes the author and affiliation and title
+%% information and builds the first part of the formatted document.
+\maketitle
+
+\section{Introduction}
+Learning and acquiring knowledge in the human world often rely on supervision through labeled data. However, there exist certain tasks, such as natural language inference (NLI), where humans can perform well without supervision. In this case, individuals can differentiate between contradictory events, such as “\textit{a woman is brushing her teeth}” and “\textit{a woman is playing the piano}”, without the use of NLI labels. Interestingly, this unsupervised learning process still ensures the acquisition of knowledge.
+
+Natural language understanding (NLU) is a critical component of many language tasks, including NLI and question-answering. While supervised pre-trained language models trained on large-scale datasets have demonstrated competitive performance on such tasks, it is important to note that humans predominantly acquire knowledge in an unsupervised manner. Despite the lack of supervision, there exist other modalities, particularly the visual modality, that can provide us with supervised signals. For example, we possess the commonsense knowledge that brushing teeth and playing the piano are mutually exclusive events, but such knowledge is not easy to acquire through text alone. However, the presence of images can readily convey this contradictory relationship. Therefore, it is reasonable to posit that combining vision and language would be advantageous for NLU.
+\begin{figure}
+	\centering 
+	\includegraphics[scale=0.35]{image/motivation.pdf} 
+	\caption{Our goal is to jointly exploit the information of text and image, and the text can be influenced by the image in back-propagation through the red line. In addition, we hope to pull positive samples closer and push negative samples apart in feature space.} 
+	\label{Fig.main} 
+\end{figure}
+Previous studies \cite{kenton2019bert, liu2019roberta, yang2019xlnet} have demonstrated the efficacy of proxy tasks, such as masked language modeling in Bert\cite{kenton2019bert}, for single modality pre-trained without labeled data, leading to improved sentence representations for downstream tasks. However, recent studies \cite{zhang2020neural, zhao2020visually, bordes2019incorporating} have found that incorporating vision as a modality can improve language model grounding and enhance performance on NLU tasks. Additionally, studies \cite{radford2021learning, jia2021scaling, yuan2021florence} have shown that using both language and vision as inputs can adapt pre-trained models to a wider range of downstream tasks, including image-text retrieval and zero-shot image classification. \cite{cui2020unsupervised} proposes using pre-trained models with vision and language as inputs for downstream pure-text tasks to supervise language with vision, although it lacks fusion information and the ability to distinguish difficult samples. In another study \cite{zhang2022mcse}, a dropout mask method for data augmentation is employed to effectively identify indistinguishable positive and negative samples, but it did not consider image-text alignment. In contrast, our approach aims to alleviate these problems by aligning different modalities and pulling positive samples closer while pushing negative samples apart, as illustrated in Fig. \ref{Fig.main}.
+
+In this work, we present a visually-supervised pre-trained multimodal model, which incorporates both text and image inputs during pre-training, and focuses on plain text tasks as the downstream task. The main objective of the model is to force the text representation to be close to its corresponding image and to align each token with a homologous patch in the image. To achieve this, the model employs global and local image-text contrastive learning, text swapping task, image-text matching task and data-based contrastive learning. To improve the discriminative ability of the text encoder in identifying positive samples from highly similar negative samples, the study applies an augmented data contrastive learning method. Rather than performing real data augmentation, we input the same data into different encoders to obtain highly similar but not identical features. The contributions of this study are summarized as follows: (i) we propose a visually-supervised pre-trained multimodal model, termed ViPMM. It outperfroms single modality pre-trained model and multimodal pre-trained model on seven pure-text and sentiment classification tasks; (ii) we employ a data-augmented contrastive learning method that uses both dynamically updated momentum encoder and tucker fusion learn effective features to enhance the discriminative ability of the text encoder in identifying positive samples from highly similar negative samples; (iii) we propose text swapping task, which exchanges the text in an image-text pair with another in a certain probability to compute bidirectional hinge similarity loss; (iv) we maximize the relevance score of the aligned token-image pairs over unaligned pairs, thus optimizing the model parameters to align tokens with their corresponding image patches. We perform the pre-training on image-text dataset, Flickr-30k. To verify the effectiveness of our method, we choose seven natural language understanding tasks and one multimodal sentiment classification task as downstream tasks for unsupervised evaluation. Experimental results demonstrate the effectiveness of our approach, where competitive performance is achieved on all tasks.
+
+\section{Related Work}
+
+\subsection{Language Pre-training}
+In the field of natural language processing, language pre-trained models have been widely studied as a means of improving sentence representation learning. These models can be broadly classified into two categories: supervised\cite{wieting2020bilingual, reimers2019sentence, cer2018universal, conneau2017supervised} and unsupervised\cite{gao2021simcse, kenton2019bert, liu2019roberta, yang2019xlnet} approaches. While supervised methods rely on labeled natural language data, unsupervised methods leverage intrinsic semantic information to learn sentence representation. Recently, self-supervised pre-training language models have garnered much attention. Such models rely on various proxy tasks to bring positive samples closer and push negative samples apart. For instance, Bert\cite{kenton2019bert} performs masked word prediction and next sentence prediction tasks. Other approaches, such as those proposed by \cite{liu2021self}, categorize self-supervised learning into contrastive, generative, and generative-contrastive methods. Meanwhile,  \cite{gao2021simcse} proposes the use of a dropout mask to augment data without real augmentation, thereby achieving a contrastive objective. However, it is worth noting that these approaches only consider language without taking into account the potential auxiliary effects of other modalities on language
+
+\subsection{Visually-supervised Pre-training}
+\begin{figure*}
+	\centering 
+	\includegraphics[scale=0.70]{image/glo.pdf} 
+	\caption{Detailed illustration of our model architecture. All features are involved in multimodal proxy tasks module.}  
+	\label{global} 
+\end{figure*}
+In the realm of representation learning, multimodality has emerged as a significant trend. In the context of natural language self-supervised learning, sentence representation can be effectively learned with supervision from other modalities, including visual and audio information. To this end, various novel approaches have been proposed. \cite{tan2020vokenization} introduces the concept of “voken”, which is akin to tokens, and utilizes visually-supervised methods to learn sentence representation. \cite{radford2021learning} proposes image-text contrastive learning, while \cite{zhang2022mcse} builds upon the work of \cite{gao2021simcse} and designs a multimodal contrastive objective. \cite{cui2020unsupervised}, on the other hand, puts forward a decoupled text-image encoder, which encodes text and image separately while ensuring full interaction between the two modalities through local mutual maximization. Lastly,  \cite{bordes2019incorporating} embeds text and image into a common space and employs a technique that pulls matched pairs closer and pushes mismatched pairs apart. However, while these methods offer promising results, they either fail to align language with vision or do not fully leverage the complementary information between the modalities, thus struggling to distinguish positive samples from highly similar negative samples.
+
+\subsection{Momentum Encoder}
+
+Various pioneering approaches have been proposed to address the problem of difficult samples in contrastive learning. \cite{he2020momentum} puts forward a novel technique called momentum encoder, which involves manually updating the encoder's parameters to encode augmented data as negative samples. Moreover, \cite{li2021align}  improves learning under noisy supervision by employing momentum distillation with momentum encoder. In our approach, we introduce a dynamic parameters update mechanism that adjusts the weight of parameter updates based on the similarity between data and augmented data. This allows us to make augmented features more similar to the original features and enhance the model's discriminative ability.
+
+\section{The Proposed Approach}
+
+In this section, we present our proposed approach and the multimodal proxy tasks we designed. Section 3.1 showcases the use of Image-Text Contrastive(ITC) to maximize global mutual information (MI). Section 3.2 introduces a local MI maximization method for local contrastive learning. Section 3.3 elaborates on Text Swapping(TS) and Image-Text Matching(ITM) as proxy tasks. In section 3.4, we discuss the utilization of augmented data contrastive learning. We compute the final loss in section 3.5. The model architecture is depicted in Fig. \ref{global}.
+
+\subsection{Image-Text Contrastive Learning}
+In this section, we introduce InfoNCE loss\cite{van2018representation} for image-text contrastive learning from the perspective of mutual information maximization.
+Mutual information has become a trend to measure the correspondence between modalities. In the case of cross-modal self-supervised learning (SSL)\cite{hjelmlearning}, we intuitively think that the higher mutual information between text and image, the better they match. So multimodal representation learning is to maximize mutual information $ {\mathcal I(X,Y)} $ between one modality $X$ and another modality $Y$,
+
+\begin{equation}\label{1}
+	{\mathcal I(X,Y)={\sum\limits}_{y\in Y}{\sum\limits}_{x\in X}P(x,y)\log \frac {P(x|y)}{P(x)}},
+\end{equation}
+where $x$, $y$ is modality feature from $X$ and $Y$.
+
+From Eqn. \ref{1}, we see that if $x$ and $y$ are incompatible with each other, $ \frac{P(x \mid y)}{P(x)}$ is 0. Hence, $ \frac{P(x \mid y)}{P(x)}$ is proportional to the similarity of $x$ and $y$. The term “$glo$” is utilized to denote the representation learning process that pertains an entire text or image, as a means of distinguishing it from the local structures. Because computing $ \frac{P(x \mid y)}{P(x)}$ is hard, we use the function $ \phi_{glo}(x,y) $ to model it in Eqn. \ref{1},
+
+\begin{equation}\label{2}
+	\phi_{glo}(x,y)\propto\frac{P(x|y)}{P(x)},
+\end{equation}
+where $\phi_{glo}(x,y)$ is a measurement of similarity and can not be normalized.
+
+%In Eqn. (2), $\phi_{glo}(x,y)$ is proportional to $ \frac{P(x \mid y)}{P(x)}$, so we model $ \frac{P(x \mid y)}{P(x)}$ by cross-modal similarity. To compute that, we use text encoder and image encoder encode $x$, $y$ to $\mathcal X$ and $\mathcal Y$. Then, processed to $\mathcal X^{norm}$ and $\mathcal Y^{norm}$ through L2-normalization layer. We follow \cite{misra2020self} to compute $\phi_{glo}(x,y)$ with exponential function with cosine similarity,
+In Eqn. \ref{2}, we propose a proportionality between $\phi_{glo}(x,y)$ and $ \frac{P(x \mid y)}{P(x)}$. We aim to model $ \frac{P(x \mid y)}{P(x)}$ using cross-modal similarity, wherein both text and image encoders are utilized to encode $x$ and $y$ into $\mathcal X$ and $\mathcal Y$. The encoded representations are then normalized through an L2-normalization layer, resulting in $\mathcal X_{norm}$ and $\mathcal Y_{norm}$. We adopt the approach proposed in \cite{misra2020self} to compute $\phi_{glo}(x,y)$ using the exponential function with cosine similarity,
+\begin{equation}\label{5}
+	\begin{split}
+		\phi_{glo}(x,y)&=d(\frac{\mathcal X}{||\mathcal X||_2},\frac{\mathcal Y}{||\mathcal Y||_2})\\
+		&=\exp (\frac{\text{cosine}(\mathcal X_{norm},\mathcal Y_{norm})}{\tau_{\phi}}),
+	\end{split}
+\end{equation}
+where $\tau_{\phi}$ is a temperature hyper-parameter.
+
+To maximize mutual information, we adpot InfoNCE based Noise-Contrastive Estimation (NCE)\cite{van2018representation} to estimate $\phi_{glo}(x,y)$, which is defined as,
+\begin{equation}\label{3}
+	\begin{split}
+		\mathcal L^\text{NCE}_{P(y|x)}(x,y;\phi_{glo})=-\mathbb{E}_{f(x,y)}(\\
+		\frac{\phi_{glo}(x, y)}{{\sum\limits}_{y^{'} \sim P(y)}\phi_{glo}(x, y^{'})}),
+	\end{split}
+\end{equation}
+where $f(x,y)$ denotes [$x,y \sim P(y|x)P(x)$]. %$P(x)$ is real distribution of $x$, $P(y|x)P(x)$ denotes the distribution of $y$ given $x$, and $P(y)$ represents negative samples distribution of $y$. Eqn. (4). can be seen from negative samples identifying positive image $y \sim P(y|x)$ given $x$.
+Here, we define $P(x)$ as the real distribution of $x$, $P(y|x)P(x)$ as the distribution of $y$ given $x$, and $P(y)$ as the distribution of negative samples of $y$. Eqn. \ref{3} describes a methodology for identifying positive images by leveraging negative images. Specifically, given $x$, the distribution $y \sim P(y|x)$ is utilized to identify positive images.
+
+The crux of our approach towards maximizing mutual information involves the utilization of an alternative technique for maximizing its lower bound,
+\begin{equation}
+	{\mathcal I(X,Y)}\geq \log N^{'}-\mathcal L^\text{NCE}(x,y;\phi),
+\end{equation}
+where $N{'}$ is number of negative samples. According to \cite{van2018representation}, minimizing $\mathcal L^\text{NCE}$ is equivalent to maximizing the lower bound of mutual information.
+
+Symmetrically, we also need to identify positive texts $x \sim P(x|y)$ given positive images $y$ from negative texts $x^{'} \sim P(x)$,
+\begin{equation}\label{4}
+	\begin{split}
+		\mathcal L^\text{NCE}_{P(x|y)}(x,y;\phi_{glo})\!\!=\!\!-\mathbb{E}_{f(y,x)}(\\
+		\frac{\phi_{glo}(x, y)}{{\sum\limits}_{x^{'} \sim P(x)}\phi_{glo}(x^{'}, y)}),
+	\end{split}
+\end{equation}
+where $f(y,x)$ denotes [$x, y\sim P(x|y)P(y)$].
+
+Combining Eqn. \ref{3} and Eqn. \ref{4}, we get image-text contrastive loss for global mutual information maximization,
+\begin{equation}\label{6}
+	\mathcal L_{itc}\!=\!\mathcal L^\text{NCE}_{P(y|x)}(x,y;\phi_{glo})+\mathcal L^\text{NCE}_{P(x|y)}(x,y;\phi_{glo}).
+\end{equation}
+
+\subsection{Local Image-Text Contrastive Learning}
+%In this section, we elaborate $\phi_{local}$ for local structures and maximize local mutual information.
+
+%Unalignment between image and text is a big challenge for representation learning. To solve this problem, we use the attention mechanism to align modalities and interact among local structures in Fig. 4.
+%In this section, we present an elaboration of $\phi_{local}$, which pertains to local structures, and propose a technique that maximizes local mutual information. 
+In section 3.1, we elaborate the definition of $\phi_{glo}$, similiarly, this section we present an introduction of $\phi_{loc}$, which primarily deals with local structures. Additionally, we propose a novel technique that facilitates the maximization of local mutual information.
+
+The issue of misalignment between image and text represents a substantial challenge in representation learning. To address this challenge, we leverage the attention mechanism to align modalities and foster interaction among local structures as depicted in Fig. \ref{local}.
+
+%For a sentence containing $\mathcal V$ words $(s^{(1)}, ...,  s^{(\mathcal V)})$, we utilize $(\mathcal S^{(1)}, ..., \mathcal S^{(\mathcal V)})$ to represent local features encoded by text encoder. Moreover, for an image with $\mathcal M\times\mathcal M$ patches ($p^{(1)} ......p^{(\mathcal M^{2})}$), local features is represented as ($\mathcal P^{(1)}, ..., \mathcal P^{(\mathcal M^{2})}$). We compute an attention map from the local features of text as well as image to achieve inter-modal alignment,
+In the case of a sentence comprising of $V$ words $(s^{(1)}, ..., s^{(V)})$, we leverage $(\mathcal S^{(1)}, ..., \mathcal S^{(V)})$ as a representation for the local features encoded by the text encoder. Similarly, for an image consisting of $ M \times M$ patches $(p^{(1)} ......p^{(M^{2})})$, local features are represented as $(\mathcal P^{(1)}, ..., \mathcal P^{(M^{2})})$. By computing an attention map between the local features of both text and image, we aim to achieve inter-modal alignment,
+%\begin{figure*} \centering    
+%	\subfigure[figure 1 title.] {
+	%		\label{fig:a}     
+	%		\includegraphics[scale=0.75]{image/local.pdf}  
+	%	}     
+%	\subfigure[figure 2 title.] { 
+	%		\label{fig:b}     
+	%		\includegraphics[scale=0.65]{image/tucker_fusion.pdf}     
+	%	}    
+%	\caption{ general title. }     
+%	\label{fig}     
+%\end{figure*}
+\begin{figure*}[!htb]
+	\centering
+	\begin{subfigure}[MI maximization for local structures. $V$  denotes length of sentence. $M\times M$ denotes the number of image patches.]{\includegraphics[width=0.64\linewidth]{image/local.pdf}
+			\label{local}
+		} 
+	\end{subfigure}
+	\quad
+	\begin{subfigure}[The structure of tucker fusion.]{\includegraphics[width=0.31\linewidth]{image/tucker_fusion.pdf}
+			\label{tucker}
+		} 
+		\caption{The structure of patch-word alignment and tucker fusion.}
+	\end{subfigure}
+	\vspace{0.2in}
+\end{figure*}
+%\begin{figure*}
+%	\centering 
+%	\includegraphics[scale=0.76]{image/图7.pdf} 
+%	\caption{MI maximization for local structures. $\mathcal V$  denotes length of sentence. $\mathcal M\times\mathcal M$ denotes the number of image patch.} 
+%	\label{local} 
+%\end{figure*}
+\begin{subequations}
+	\begin{align}
+		attn_{i,j}=\frac{\exp({\mathcal S^{(i)}}^T\cdot \mathcal P^{(j)})}{{\sum\limits}_k\exp({\mathcal S^{(i)}}^T\cdot \mathcal P^{(k)})},\\
+		attn^{'}_{i,j}=\frac{\exp({\mathcal S^{(i)}}^T\cdot \mathcal P^{(j)})}{{\sum\limits}_k\exp({\mathcal S^{(k)}}^T\cdot \mathcal P^{(j)})},
+	\end{align}
+\end{subequations}
+where $k$ is index of negative samples. $attn_{i,j}$ denotes the attention of $i$-th word to $j$-th patch and $attn^{'}_{i,j}$ denotes the attention of $j$-th patch to $i$-th word.
+
+According to attention map above, we assign weights to features,
+
+\begin{subequations}
+	\begin{align}
+		\mathcal S^{'}=\frac{\exp(attn_{i,j}/\tau_{1})}{{\sum\limits}_k \exp(attn_{k,j}/\tau_{1})}\mathcal S,\\
+		\mathcal P^{'}=\frac{\exp(attn^{'}_{i,j}/\tau_{1})}{{\sum\limits}_k \exp(attn^{'}_{i,k}/\tau_{1})}\mathcal P,
+	\end{align}
+\end{subequations}
+where $\tau_1$ denotes the temperature hyper-parameter. Then we compute the alignment score by Eqn. \ref{5},
+\begin{subequations}
+	\begin{align}
+		\phi_{loc}(s,p^{(j)})=d(\mathcal {P^{'}}^{(j)}, \mathcal S^{'}),\\
+		\phi_{loc}(s^{(i)},p)=d(\mathcal {S^{'}}^{(i)}, \mathcal P^{'}).
+	\end{align}
+\end{subequations}
+
+
+Combining $\phi_{loc}(s,p^{(j)})$ and $\phi_{loc}(s^{(i)},p)$, we obtain $\phi_{loc}(s,p)$ function,
+\begin{equation}
+	\begin{split}
+		\phi_{loc}(s,p)=&\sum\limits_{i=1}^{V}\phi_{loc}(s^{(i)},p)\\
+		&+\sum\limits_{j=1}^{M^2}\phi_{loc}(s,p^{(j)}).
+	\end{split}
+\end{equation}
+
+According to Eqn. \ref{3}, Eqn. \ref{4} and Eqn. \ref{6}, we can compute local image-text contrastive loss $\mathcal L_{loc}$ for local mutual information maximization,
+\begin{equation}
+	\mathcal L_{loc}\!=\!\mathcal L^\text{NCE}_{P(p|s)}(s,p;\phi_{loc})+\mathcal L^\text{NCE}_{P(s|p)}(s,p;\phi_{loc}).
+\end{equation}
+
+
+\subsection{Image-Text Matching and Text Swapping}
+%Lying in the propose of reducing the distance between image and text in feature space, we design two tasks to match them: (1) with a certain probability randomly exchange the text corresponding to current image with another text, see whether the model can accurately discriminate and (2) compute relevance score between token and image to match them while considering this sentence as a context, which brings text and image closer in a more granular method.
+Lying in the propose of minimizing the semantic gap between vision and language in feature space, we present two tasks for their alignment: (1) random text swapping with a predetermined probability, followed by assessment of the model's discriminative ability. (2) computation of relevance scores between image tokens and contextualized sentences, resulting in a more nuanced approach for enhancing the text-image relationship.\\
+\textbf{Image-Text Matching}. %Inspired by \cite{tan2020vokenization}, we design a proxy task for image-text matching, with the purpose of matching each token in a sentence with an image. So we tokenize a sentence, and then compute the relevance score of each token and image. %We model relevance score with an inner product of image feature representation $\mathcal J_y$ and local text feature representation $\mathcal J_s$,
+Inspired by \cite{tan2020vokenization}, we propose a proxy task for aligning images with their corresponding textual content at a token level. To this end, we first tokenize the text and subsequently compute the relevance scores between each token and the corresponding image. Because image and text features are mapped into a shared space, the relevance score is modeled as an inner product between the global image feature representation $\mathcal J_y$ and the local text feature representation $\mathcal J_s$,
+\begin{equation}\label{7}
+	r(\mathcal S,\mathcal Y)=\mathcal J_y^T\cdot \mathcal J_s.
+\end{equation}
+
+Before computing relevance score, we apply a multi-layer perceptron (MLP) $\text{MLP}_t$ to down project local text feature $\mathcal S$, an L2-normalization layer is applied subsequently. Similarily, for image feature $\mathcal Y$ we perform the same operation with $\text{MLP}_v$,
+\begin{subequations}
+	\begin{align}
+		\mathcal J_s=\frac{\text{MLP}_t(\mathcal S)}{||\text{MLP}_t(\mathcal S)||_2},\\
+		\mathcal J_y=\frac{\text{MLP}_v(\mathcal Y)}{||\text{MLP}_v(\mathcal Y)||_2}.
+	\end{align}
+\end{subequations}
+
+According to Eqn. \ref{7}, we calculate the relevance score $r(\mathcal S,\mathcal Y)$ for a positive token-image pair. Subsequently, we randomly select another image $\mathcal Y^{'}$ and obtain its corresponding relevance score $r(\mathcal S,\mathcal Y^{'})$. To ensure that the difference between $r(\mathcal S,\mathcal Y)$ and $r(\mathcal S,\mathcal Y^{'})$ is greater than a pre-specified margin $M$, we utilize the hinge loss function to compute image-text match loss,
+\begin{equation}
+	\mathcal L_{itm}=max(0, M-r(\mathcal S,\mathcal Y)+r(\mathcal S,\mathcal Y^{'})).
+\end{equation}
+\textbf{Text Swapping}. We present an additional objective for attaining image-text match, which we refer to as “Text Swapping”. This involves randomly substituting text with a predefined probability, and employing a bidirectional similarity hinge loss to penalize the model for inadequate discrimination capability. 
+%\begin{figure*}[htbp]
+%	\centering
+%	\subfigure[pic1.]{
+	%		\begin{minipage}[t]{0.33\linewidth}
+		%			\centering
+		%			\includegraphics[width=2in]{image/图4.pdf}
+		%			%\caption{fig1}
+		%		\end{minipage}%
+	%	}%
+%	\subfigure[pic2.]{
+	%		\begin{minipage}[t]{0.33\linewidth}
+		%			\centering
+		%			\includegraphics[width=2.5in]{image/图3.pdf}
+		%			%\caption{fig2}
+		%		\end{minipage}%
+	%	}%
+%	\subfigure[pic3.]{
+	%		\begin{minipage}[t]{0.33\linewidth}
+		%			\centering
+		%			\includegraphics[width=1.5in]{image/图6.pdf}
+		%			%\caption{fig2}
+		%		\end{minipage}
+	%	}%
+%	\centering
+%	\caption{ pics}
+%\end{figure*}
+Initially, we utilize an L2-normalization layer to normalize both the image feature $\mathcal Y$ and text feature $\mathcal X$,
+\begin{subequations}
+	\begin{align}
+		\mathcal X_{norm}=\frac{\mathcal X}{||\mathcal X||_2},\\
+		\mathcal Y_{norm}=\frac{\mathcal Y}{||\mathcal Y||_2}.
+	\end{align}
+\end{subequations}
+
+We propose to fuse features from different modalities using cross-modal attention mechanisms,
+\begin{subequations}
+	\begin{align}
+		&attn_{x2y}=\frac{\mathcal Y_{norm}\cdot \mathcal X_{norm}^T}{\sqrt{d}},\\
+		\mathcal F_x&=\text{Softmax}(attn_{x2y})\cdot \mathcal X_{norm},\\
+		&attn_{y2x}=\frac{\mathcal X_{norm}\cdot \mathcal Y_{norm}^T}{\sqrt{d}},\\
+		\mathcal F_y&=\text{Softmax}(attn_{y2x})\cdot \mathcal Y_{norm},
+	\end{align}
+\end{subequations}
+where $d$ is the feature dimension of $\mathcal X_{norm}$ and $\mathcal Y_{norm}$.
+
+In order to measure similarity between text and image, we adopt a two-way linear similarity combination method,
+\begin{subequations}
+	\begin{align}
+		g=(\mathcal X_{norm})^T\mathcal Y_{norm}+\alpha\cdot (\mathcal F_x)^T\mathcal F_y, \label{8}\\
+		g^{'}=(\mathcal X^{'}_{norm})^T\mathcal Y_{norm}+\alpha\cdot (\mathcal F^{'}_x)^T\mathcal F_y \label{9},
+	\end{align}
+\end{subequations}
+where $\mathcal X^{'}_{norm}$ is a normalized textual feature  which does not align with the image feature $\mathcal Y$. The fused feature of $\mathcal X^{'}_{norm}$ and $\mathcal Y$ is denoted by $\mathcal F^{'}_x$. The scalar value $\alpha$ is treated as a constant in this context for weights balance.
+
+Utilizing Eqn. \ref{8} and Eqn. \ref{9}, we can compute text swapping task loss,
+\begin{equation}
+	\mathcal L_{ts}=max(0, M-g+g^{'}).
+\end{equation}
+
+\subsection{Augmented Data Contrastive Learning}
+In propose to focus on effective and complementary features, we apply augmented data contrastive learning inspired by \cite{li2022clmlf}. 
+
+%Differently, since our downstream tasks are plain text tasks, proxy tasks focus on text. However, here we do not perform traditional data augmentation methods such as back translation, instead, we are inspired by \cite{gao2021simcse, he2020momentum}, apply momentum encoder to encode original data as augmented data. Meanwhile, we maintain a first-in-first-out queue to store negative samples. For image data, we perform random transformation to do augmentation encoded by momentum encoder, and we also maintain a same queue to store negative samples initialized randomly. Furthermore, in order to utilize complementary information between modalities, we perform modal fusion via tensor approach, which guarantees data loss to be minimal to some extent.
+
+
+In contrast to previous approaches, we focus on proxy tasks that center on text since our downstream tasks involve plain text. To augment the data, we adopt a momentum encoder approach inspired by \cite{gao2021simcse, he2020momentum} rather than conventional methods such as back translation. Specifically, we encode original data using a momentum encoder to generate augmented data. To handle negative samples, we use a first-in-first-out queue. As for image data, we apply random transformations using a momentum encoder for image augmentation, and we also initialize a queue to store negative samples randomly. To take advantage of the complementary information across modalities, we perform modal fusion through a tensor approach that minimizes data loss to some extent.\\
+\textbf{Data-based Contrastive Learning by Momentum Encoder}. %Instead of applying traditional data augmentation method, we are inspired by \cite{gao2021simcse} to utilize an approch similar to dropout mask. We design two momentum encoders initialized by image encoder and text encoder respectively, whose parameters are not affected by the back-propagation of gradient, but manually combined with the modality encoder's parameters linearly to perform dynamic or static update by ourselves. Since back-translation takes a lot of time, we input the same data into encoders (modality encoder and corresponding momentum encoder) with different parameters, so that the obtained data is similar but different.
+Rather than using conventional data augmentation methods, we adopt an approach similar to dropout masks, inspired by \cite{gao2021simcse}. We construct two momentum encoders initialized by our image encoder and text encoder, respectively. These momentum encoders are designed to have their parameters not affected by the back-propagation of gradients, but are instead manually combined linearly with the common modality encoder's parameters for either dynamic or static updates. For static updates, we predefine an $m$ as momentum for updates, and dynamic updates is by computing the similiarity of data and augmented data as momentum. To avoid the time-consuming nature of back-translation, we input the same data into both the modality encoder and the corresponding momentum encoder with different parameters, resulting in similar but distinct representations of the data.
+
+%Inspired by \cite{he2020momentum}, in order to reduce memory usage, we set two first-in-first-out queues for text and image respectively, which is randomly initialized used to store negative samples. The features output by momentum encoder as negative samples are enqueued from tail, and the earliest batch samples are dequeued from head. Algorithm 1 describes the process of above. We compute $\mathcal L_t$ and $\mathcal L_v$ respectively according to Algorithm 1.
+Drawing inspiration from \cite{he2020momentum}, we implement a memory-efficient strategy by creating two first-in-first-out queues for text and image, respectively. These queues are randomly initialized and used to store negative samples. The momentum encoder generates features that serve as negative samples, which are then enqueued from the tail of the queue. The earliest batch of samples is dequeued from the head. Algorithm. \ref{alg} provides a detailed overview of this process. We compute text and augmented image contrastive loss $\mathcal L_{t2v}$ and image and augmented text contrastive loss $\mathcal L_{v2t}$ according to Algorithm. \ref{alg}.
+\begin{algorithm}
+%	\begin{spacing}{0.8}
+	\caption{Augmented data contrastive learning} \label{alg}
+		\begin{algorithmic}
+		\STATE \textcolor[RGB]{0,90,20}{\# f\_o, f\_m: original and momentum encoder }
+		\STATE \textcolor[RGB]{0,90,20}{\# q: a queue stores negative samples }
+		\STATE \textcolor[RGB]{0,90,20}{\# m, $\tau$: momentum and temperature }
+		\STATE \textcolor[RGB]{0,90,20}{\# k: size of queue}
+		\STATE f\_m.param = f\_o.param \quad \textcolor[RGB]{0,90,20}{\# initialize}
+		\STATE $f$ = f\_o($x$) \quad \textcolor[RGB]{0,90,20}{\# $f$: encoded feature, $x$: data}
+		\STATE $f_{aug}$ = f\_m($x$) \quad \textcolor[RGB]{0,90,20}{\# $f_{aug}$: augmented feature}
+		\STATE l\_pos = $f \times f_{aug}$ \quad \textcolor[RGB]{0,90,20}{\# positive logits(dx1)}
+		\STATE l\_neg = $f \times$ q \quad \textcolor[RGB]{0,90,20}{\# negative logits(dxk)}
+		\STATE logits = cat([l\_pos, l\_neg], dim=-1)  \quad \textcolor[RGB]{0,90,20}{\# dx(k+1)}
+		\STATE label = zeros(d) \quad \textcolor[RGB]{0,90,20}{\# positives are the 0-th}
+		\STATE loss = CrossEntropy(logits/$\tau$, label) \quad \textcolor[RGB]{0,90,20}{\# contrastive loss}
+		\STATE loss.backward() \quad \textcolor[RGB]{0,90,20}{\# backward}
+		\STATE \textcolor[RGB]{0,90,20}{\# parameter updating}
+		\STATE f\_m.param = m*f\_o.param+(1-m)*f\_m.param
+		\STATE enqueue(q, l\_neg) \quad \textcolor[RGB]{0,90,20}{\# enqueue current batch}
+		\STATE dequeue(q) \quad \textcolor[RGB]{0,90,20}{\# dequeue from head}
+		\end{algorithmic}
+%	\end{spacing}
+\end{algorithm}\\
+\textbf{Data-based Contrastive Learning by Tucker Fusion}. We present a fused data contrastive learning. Modal fusion has the following advantages: (1) it is more robust than a single mode, (2) the modal information is complementary and (3) it can run without any modal information. Inspired by \cite{ben2017mutan}, we apply Multimodal Tucker Fusion, shown in Fig. \ref{tucker}, to merge text and image for further contrastive learning,
+\begin{equation}
+	{\mathcal T}=(({\mathcal T_c}\times_{1} \mathcal W_e)\times_{2}\mathcal W_{u})\times_3\mathcal W_{o}, \label{tk}
+\end{equation}
+where ${\mathcal T_c}$  is core tensor, and $W_e$, $W_u$ and $W_o$ are all factor matrices.
+
+Bilinear models, as described in \cite{fukui2016multimodal, kim2016hadamard}, offer a solution to the fusion problem by encoding bilinear interactions between any two feature vectors $z$ and $h$,
+\begin{equation}
+	\omega=(({\mathcal T_c}\times_{1}z)\times_{2}h).
+\end{equation}
+
+We follow Eqn. \ref{tk}, treat $W_o$ as a mapping matrix which is trainable and maps fused features to a certain dimensional space,
+\begin{subequations}
+	\begin{align}
+		&\mathcal Q=(({\mathcal T_c}\times_{1}{\mathcal X})\times_{2}{\mathcal Y})\times_{3}\mathcal W_o,\\
+		\mathcal Q_{aug}&=(({\mathcal T_c}\times_{1}{\mathcal X_{aug}})\times_{2}{\mathcal Y_{aug}})\times_{3}\mathcal W_o,
+	\end{align}
+\end{subequations}
+where ${\mathcal X_{aug}}$, ${\mathcal Y_{aug}}$ are augmented ones. In addition, $\mathcal Q$ is fused feature and $\mathcal Q_{aug}$ denotes augmented fused feature.
+
+Then, we compute contrastive loss via CrossEntropy loss,
+\begin{equation}
+	\mathcal L_q \!=\!\text{CrossEntropy}(\mathcal Q\cdot{(\mathcal Q^{aug})}^T/\tau_2,label),
+\end{equation}
+where $\tau_2$ is temperature hyper-parameter, and $label \in [0, N-1]$, $N$ is the batch size. The final loss $\mathcal L_{adcl}$ is computed as,
+\begin{equation}
+	{\mathcal L_{adcl}}={\mathcal L_{t2v}} + {\mathcal L_{v2t}} + {\mathcal L_q}.
+\end{equation}
+\subsection{Model Training}
+All forementioned loss can be simply added to be final loss,
+\begin{equation}
+	\mathcal L = \beta\mathcal L_{itc}+\gamma\mathcal L_{itm}+\eta\mathcal L_{ts}+\mu\mathcal L_{adcl} + \sigma\mathcal L_{loc},
+\end{equation}
+where $\beta$, $\gamma$, $\eta$, $\mu$ and $\sigma$ are hyper-parameters for weights balance.
+
+\section{Experimental Settings}
+\subsection{Downstream Tasks}
+For downstream tasks, we choose seven GLUE tasks for evaluation as follows:\\
+\textbf{MRPC} (The Microsoft Research Paraphrase Corpus), a similarity and paraphrase task, is to automatically extract a sentence pair corpus from online news sources, and manually annotate whether the sentences in the sentence pair are semantically equivalent.\\
+\textbf{QQP} (The Quora Question Pairs), a similarity and paraphrase task, is a collection of question pairs in the community question-and-answer website Quora. The task is to determine whether a pair of questions are semantically equivalent.\\
+\textbf{MNLI} (The Multi-Genre Natural Language Inference Corpus), a natural language inference task, given a premise statement and a hypothesis statement, the task is to predict whether the premise statement contains a hypothesis entailment), contradicts the hypothesis (contradiction) or both neither (neutral). And MNLI-mm is MNLI with mismatched validation set.\\
+\textbf{QNLI} (Qusetion-answering NLI), natural language inference task.The task is to judge whether the questions and sentences imply, imply and not imply, and classify them into two categories.\\
+\textbf{RTE} (The Recognizing Textual Entailment datasets), a natural language inference task, the task is to judge whether a sentence pair contains, whether sentence 1 and sentence 2 contain each other, and a binary classification task.\\
+\textbf{SNLI} (The Stanford Natural Language Inference), is a collection of human-written English sentence pairs, with manually labeled categories entailment, contradiction, and neutral. The statistics of these datasets are shown in Table. \ref{data}.\\
+\begin{table}
+	\centering
+	\setlength{\tabcolsep}{1mm}{
+		\resizebox{\linewidth}{!}{
+			\begin{tabular}{ccccc}
+				\toprule
+				Dataset & Type &Train & Dev & Test\\
+				\midrule
+				Flickr30k & Text2Image & 111,240 & 47,675 & -\\
+				RTE & NLI & 2,491 & 277 & 3,000\\
+				SNLI & NLI &550,152 & 10,000 & 10,000\\
+				MRPC &NLI &3,668 & 408 & 1,725\\ 
+				MNLI & NLI &392,702 & 9,815 & 9,796\\  	
+				MNLI-mm & NLI& 392,702 & 9,832 & 9,847\\ 	
+				QQP & QA &363,870 & 40,431 & 390,965\\
+				QNLI & QA & 104,743 & 5,463 & 5,461\\
+				\bottomrule
+		\end{tabular}}
+	}
+	\caption{Statistics of datasets.}
+	\label{data}
+\end{table}
+%In addition, we perform \textbf{MSC} (Multimodal Sentiment Classification) on MOSEI dataset with one or two modalities, which consists of 23,453 utterance videos. It contains two tasks: sentiment analysis and emotion recognition.
+Besides, we conduct multimodal sentiment classification on the MOSEI dataset with one or two modalities. This dataset includes 23,453 videos of utterances and encompasses two distinct tasks: sentiment analysis and emotion recognition.
+
+\subsection{Implementation Details}
+
+Our experiment is run over a computer with NVIDIA Geforce RTX 3090, 24G VRAM. For text encoding, we leverage the \textit{bert-base-uncased} model from the Hugging Face Transformers library, while we use the \textit{ResNet50} model as the image encoder. The local text feature representation $\mathcal S^{(i)}$ corresponds to the feature vector of the $i$-th word, whereas the local image feature $\mathcal P^{(i)}$ corresponds to the $i$-th patch, which represents the feature before the final pooling layer. In order to ensure that both the text and image modalities are embedded into the same feature space, we uniformly map the feature dimension to 768. We employ separate projection heads for text and image, both of which consist of two MLP layers with Tanh activation in the middle. This structure enables the mapping of features to a shared feature space with 128 dimensions, which is normalized prior to the computation of multimodal objectives. After tucker fusion, the fused feature dimension is 128.\\
+Moreover, we utilize a momentum encoder to encode the same data for data augmentation. This encoder is initialized by the original data encoder, and its momentum $m$ is dynamically set while the queue size $K$ is 65536. Specifically, $m$ is set to the average similarity between the text and the augmented text,
+\begin{equation}
+	m = \frac{1}{N}\text{cosine}(\mathcal X, \mathcal X^{aug}).
+\end{equation}
+
+\subsection{Hyper-parameter Settings}
+
+In this section, we provide details of the hyper-parameters used in our approach. Specifically, we set the temperature of energy function $\tau_{\phi}$ to 0.1, and the weights of the objective are mainly set to 1, while $\mu$ is set to 0.04. We further explore the influence of parameter $\mu$ on the effectiveness of the model. During self-supervised learning, we set the training epoch $n$ to 10, and the batch size $N$ is set as 128. To set the learning rate ($lr$), we employ a warm-up strategy, where we initially set the learning rate as 1e-4 and the warm-up ratio $wr$ is set to 0.1. 
+
+\section{Experimental Results}
+\subsection{Main Results}
+
+In this section, we present a comparative evaluation of our proposed pre-trained model compared with mainstream pre-trained NLU models, as well as multimodal pre-trained models, across eight downstream tasks. We evaluate our proposed model on MOSEI using only language as input, and compare it with a single-modality pre-trained model that uses language and vision as inputs. Additionally, we perform experiments with language and speech as inputs for multimodal pre-trained models. We simply concat multimodal features and map them to results through a linear classification layer. Our results, presented in Table. \ref{mosei}, reveal that our proposed model with only language outperforms single modality pre-trained models with language and vision, and outperforms multimodal pre-training models (e.g. MCSE, MACD and i-Code) on all conditions. thereby emphasizing the efficacy of our approach. Moreover, we investigate the performance of forementioned models on seven pure-language tasks without fine-tuning for unsupervised representation. As shown in Table. \ref{un}, we employ both static and dynamic momentum updating methods, and demonstrate that both methods achieve competitive performance. 
+\begin{table}[!]
+	\centering
+	\resizebox{\linewidth}{!}{
+		\begin{tabular}{c|ccc|c}
+			\hline
+			Model&Language&Speech&Vision&Acc\\
+			\hline
+			Bert(NAACL'2019)&\checkmark& &\checkmark& 54.61\\
+			Roberta(2019)& \checkmark& &\checkmark& 55.19\\
+			XLNet(NIPS'2019)& \checkmark& &\checkmark& 53.23\\
+			i-Code(2022)$\mathcal y$ & \checkmark& &\checkmark& 49.0\\
+			\hline
+			Bert(NAACL'2019)&\checkmark& & & 53.18\\
+			Roberta(2019)& \checkmark& & & 55.19\\
+			XLNet(NIPS'2019)& \checkmark& & & 54.09\\
+			MACD(EMNLP'2020)& \checkmark& & & 55.7\\
+			MCSE(NAACL'2022)& \checkmark& & & 46.06\\
+			i-Code(2022)$\mathcal y$ & \checkmark& & & 46.3\\
+			ViPMM (Ours)& \checkmark& & & \textbf{55.86}\\
+			\hline
+			Bert(NAACL'2019)& \checkmark& \checkmark& & 49.0\\
+			Roberta(2019)& \checkmark& \checkmark& & 38.61\\
+			XLNet(NIPS'2019)& \checkmark& \checkmark& & 53.87\\
+			MACD(EMNLP'2020)& \checkmark& \checkmark& & 55.7\\
+			MCSE(NAACL'2022)& \checkmark& \checkmark& & 46.06\\
+			i-Code(2022)$\mathcal y$ & \checkmark& \checkmark& & 49.2\\
+			ViPMM (Ours)& \checkmark& \checkmark& & \textbf{57.75}\\
+			\hline
+	\end{tabular}}
+	\caption{Unsupervised multimodal sentiment classification results on MOSEI. $\mathcal y$: results from \cite{yang2022code}.}
+	\label{mosei}
+\end{table}
+\begin{table}[h]
+	\centering
+	\resizebox{\linewidth}{!}{
+		\begin{tabular}{cccccccccccc}
+			\toprule
+			Model & MNLI & MNLI-mm & QNLI & SNLI & QQP & MRPC & RTE \\
+			\midrule
+			Bert(NAACL'2019) & 35.9 & 36.6 & 49.5 & 34.4 & 47.0 & 67.5 & 52.7 \\
+			Roberta(NIPS'2019) & 35.4 & 35.2 & 49.4 & 33.7 & 36.8 & 67.4 & 52.7 \\
+			XLNet(NIPS'2019) & 35.7 & 36.0 & 49.7 & 34.0 & 60.3 & 67.4 & 53.4 \\
+			MACD(EMNLP'2020) & 41.6 & 41.3 & 58.6 & 50.69 & 67.0 & 65.1 & 53.1\\
+			MCSE(NAACL'2022) & 41.1 & 42.0 & 56.1 & 41.2 & 67.3 & 68.6 & 55.7\\
+			ViPMM(Ours, static) & \textbf{42.3} & 42.0 & \textbf{60.2} & \textbf{51.2} & 67.5 & \textbf{68.4} & 56.6\\
+			ViPMM(Ours, dynamic) & 42.0 & \textbf{42.1} & 59.2 & 50.3 & \textbf{68.7} & \textbf{68.4} & \textbf{65.0}\\
+			\bottomrule
+	\end{tabular}}
+	\caption{Unsupervised natural language understanding results.}
+	\label{un}
+\end{table}
+
+\begin{table}[h]
+	\centering
+	\resizebox{\linewidth}{!}{
+		\begin{tabular}{cccccccccccc}
+			\toprule
+			& MNLI & MNLI-mm & QNLI & SNLI & QQP & MRPC & RTE \\
+			\midrule
+			w/o ADCL & 35.5 & 35.3 & 49.5 & 33.7 & 38.7 & 67.5 & 55.6 \\
+			w/o tucker fusion & 42.0 & 42.0 & 57.7 & 50.8 & 64.8 & 68.4 & 53.1\\
+			w/o text swapping & 41.7 & \textbf{42.2} & 58.3 & 50.6 & 65.2 & 67.5 & 53.4 \\
+			w/o momentum encoder & 42.0 & 41.8 & 56.8 & 50.5 & 67.0 & 66.8 & 53.0 \\
+			w/o image-text matching & 41.8 & 41.8 & 58.9 & 50.6 & 67.0 & 67.5 & 52.7\\
+			w/o local MI maximization & 41.8 & 42.1 & 58.5 & 50.0 & 65.7 & 68.1 & 55.9\\
+			ViPMM (Full)& \textbf{42.3} & 42.0 & \textbf{60.2} & \textbf{51.2} & \textbf{67.5} & \textbf{68.6} & \textbf{56.6}\\
+			\bottomrule
+	\end{tabular}}
+	\caption{Ablation results of our ViPMM.}
+	\label{tab:example}
+\end{table}
+\begin{figure*}
+	\centering
+	\begin{subfigure}[Experimental results of core tensor's rank.]{\includegraphics[width=0.35\linewidth]{image/cl.pdf}
+			\label{rank}
+		} 
+	\end{subfigure}
+	\quad
+	\begin{subfigure}[Experimental results of the weights of augmented data contrastive learning.]{\includegraphics[width=0.35\linewidth]{image/mu.pdf}
+			\label{cl}
+		} 
+		\caption{Experiment results of hyper-parameters related to tucker fusion.}
+	\end{subfigure}
+	\vspace{0.2in}
+\end{figure*}
+
+\subsection{Ablation Study}
+
+To validate the effectiveness of ViPMM with our designed proxy tasks, we conduct ablation studies on the influence of various components, including tucker fusion instead of cross-modal attention-based fusion, global and local image-text contrastive learning with momentum encoder, text swapping, image-text matching, and augmented data contrastive learning (ADCL), on the performance of our  across seven tasks, as reported in Table. \ref{tab:example}. Our results demonstrate that ViPMM outperforms other models across all tasks. ADCL plays a key role in improving our model's performance by enabling tucker fusion to fuse features with less missing information. Text swapping and image-text matching both contribute to enhancing the model's ability to reduce the distance between image and text. Momentum encoder with augmented data empowers the model to identify similar but different text-image pairs. Local image-text contrastive learning assists the model in aligning text and image, thereby preventing the learning of irrelevant information. Tucker fusion to some extent reduces the loss after information fusion, and successfully utilizes complementary information between modalities, achieving better performance compared to cross-modal attention fusion.
+
+\subsection{Visualization}
+
+In order to explore the impact of the tensor fusion method on the experiment, we display the influence of the core tensor's rank and the hyper-parameter $\mu$ of related task to the downstream tasks in Fig. \ref{rank} and Fig. \ref{cl}. As an illustration of our approach, we apply t-SNE to cluster and visualize the encoded features of MACD and ViPMM, using the RTE task as a test case. Fig. \ref{tsne} display the resulting cluster diagrams. From these diagrams, we observe that the distribution of each feature encoded by our model tends to be circular, while that of MACD tends to be elliptical. This suggests that the feature encoded by ViPMM as a pre-trained encoder is more isotropic and remains unchanged as the direction changes. 
+\begin{figure}[H]
+	\centering 
+	\includegraphics[scale=0.32]{image/VPMTM.pdf} 
+	\caption{t-SNE visualization on RTE dataset encoded by ViPMM (left) and MACD (right).} 
+	\label{tsne} 
+\end{figure}
+
+\section{Conclusion}
+
+We propose a novel multimodal training framework for learning sentence representations. Our approach includes both intra-modal and inter-modal paths with proxy tasks for visual supervision of text. Our experiments demonstrate competitive performance in seven GLUE tasks and multimodal sentiment classification without fine-tuning. Our dynamic momentum updating method improves the accuracy of discovering modalities' similarities and differences. We use local mutual information maximization to align text and image. Text swapping task improves model credibility, and augmented data contrastive learning enhances model discrimination. Our model can be directly utilized into any natural language understanding tasks as text encoder.
+
+%\begin{acks}
+%To Robert, for the bagels and explaining CMYK and color spaces.
+%\end{acks}
+
+%%
+%% The next two lines define the bibliography style to be used, and
+%% the bibliography file.
+\bibliographystyle{ACM-Reference-Format}
+\bibliography{reference.bib}
+
+\end{document}
+\endinput
+%%
+%% End of file `sample-sigplan.tex'.