\documentclass{article}
\usepackage{graphicx}
\makeatletter
\newcommand*{\limitbox}[3]{%
\begingroup
\setlength{\@tempdima}{#1}%
\setlength{\@tempdimb}{#2}%
\resizebox{%
\ifdim\width>\@tempdima\@tempdima\else\width\fi
}{!}{%
\resizebox{!}{%
\ifdim\height>\@tempdimb\@tempdimb\else\height\fi
}{#3}%
}%
\endgroup
}
\makeatother
\textwidth7cm% only for demo
\parindent0cm
\begin{document}
\rule{\textwidth}{1pt}
\begin{figure}
\centering
\limitbox{\linewidth}{\textheight}{\rule{2\textwidth}{3cm}}
\caption{a very very very very very very very very very very very very very very
very very very very very very very very very very very very very very very very very very
very very very very very very very very very very very very very very very very
very very very very very very very very very very very very very very
very very very very very very very very very very long caption}
\end{figure}
\begin{verbatim}
\usepackage{graphicx}
\makeatletter
\newcommand*{\limitbox}[3]{%
\begingroup
\setlength{\@tempdima}{#1}%
\setlength{\@tempdimb}{#2}%
\resizebox{%
\ifdim\width>\@tempdima\@tempdima\else\width\fi
}{!}{%
\resizebox{!}{%
\ifdim\height>\@tempdimb\@tempdimb\else\height\fi
}{#3}%
}%
\endgroup
}
\makeatother
\end{verbatim}
\end{document}