-----BEGIN PRIVACY-ENHANCED MESSAGE-----
Proc-Type: 2001,MIC-CLEAR
Originator-Name: webmaster@www.sec.gov
Originator-Key-Asymmetric:
 MFgwCgYEVQgBAQICAf8DSgAwRwJAW2sNKK9AVtBzYZmr6aGjlWyK3XmZv3dTINen
 TWSM7vrzLADbmYQaionwg5sDW3P6oaM5D3tdezXMm7z1T+B+twIDAQAB
MIC-Info: RSA-MD5,RSA,
 LRycqaYNKDzYxpdxEmYJUHt98vNeEz46krKCgLxYx9Y6CYwVlK5O6fvknmDBciRn
 sBxUjBydPflHS9Q+on7Vew==

<SEC-DOCUMENT>0001140361-06-008186.txt : 20060525
<SEC-HEADER>0001140361-06-008186.hdr.sgml : 20060525
<ACCEPTANCE-DATETIME>20060525175402
ACCESSION NUMBER:		0001140361-06-008186
CONFORMED SUBMISSION TYPE:	4
PUBLIC DOCUMENT COUNT:		1
CONFORMED PERIOD OF REPORT:	20060523
FILED AS OF DATE:		20060525
DATE AS OF CHANGE:		20060525

REPORTING-OWNER:	

	OWNER DATA:	
		COMPANY CONFORMED NAME:			HOLTZMAN SEYMOUR
		CENTRAL INDEX KEY:			0000904439

	FILING VALUES:
		FORM TYPE:		4
		SEC ACT:		1934 Act
		SEC FILE NUMBER:	000-15898
		FILM NUMBER:		06868183

	BUSINESS ADDRESS:	
		STREET 1:		100 N WILKES BARRE BLVD
		STREET 2:		4TH FL
		CITY:			WILKES BARRE
		STATE:			PA
		ZIP:			18702

	MAIL ADDRESS:	
		STREET 1:		C/O JEWELCOR - 4TH FLOOR
		STREET 2:		100 N WILKES BARRE BLVD
		CITY:			WILKES BARRE
		STATE:			PA
		ZIP:			18702

ISSUER:		

	COMPANY DATA:	
		COMPANY CONFORMED NAME:			CASUAL MALE RETAIL GROUP INC
		CENTRAL INDEX KEY:			0000813298
		STANDARD INDUSTRIAL CLASSIFICATION:	RETAIL-FAMILY CLOTHING STORES [5651]
		IRS NUMBER:				042623104
		STATE OF INCORPORATION:			DE
		FISCAL YEAR END:			0131

	BUSINESS ADDRESS:	
		STREET 1:		555 TURNPIKE STREET
		CITY:			CANTON
		STATE:			MA
		ZIP:			02021
		BUSINESS PHONE:		7818215900

	MAIL ADDRESS:	
		STREET 1:		555 TURNPIKE STREET
		CITY:			CANTON
		STATE:			MA
		ZIP:			02021

	FORMER COMPANY:	
		FORMER CONFORMED NAME:	DESIGNS INC
		DATE OF NAME CHANGE:	19920703
</SEC-HEADER>
<DOCUMENT>
<TYPE>4
<SEQUENCE>1
<FILENAME>doc1.xml
<DESCRIPTION>FORM 4
<TEXT>
<XML>
<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0202</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2006-05-23</periodOfReport>

    <notSubjectToSection16>0</notSubjectToSection16>

    <issuer>
        <issuerCik>0000813298</issuerCik>
        <issuerName>CASUAL MALE RETAIL GROUP INC</issuerName>
        <issuerTradingSymbol>CMRG</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0000904439</rptOwnerCik>
            <rptOwnerName>HOLTZMAN SEYMOUR</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>C/O JEWELCOR MANAGEMENT, INC.</rptOwnerStreet1>
            <rptOwnerStreet2>100 N. WILKES BARRE BLVD., 4TH FLOOR</rptOwnerStreet2>
            <rptOwnerCity>WILKES BARRE</rptOwnerCity>
            <rptOwnerState>PA</rptOwnerState>
            <rptOwnerZipCode>18702</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>1</isTenPercentOwner>
            <isOther>0</isOther>
            <officerTitle></officerTitle>
            <otherText></otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <derivativeTable>
        <derivativeTransaction>
            <securityTitle>
                <value>Warrants (right to buy)</value>
            </securityTitle>
            <conversionOrExercisePrice>
                <value>5.26</value>
            </conversionOrExercisePrice>
            <transactionDate>
                <value>2006-05-23</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>J</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
                <footnoteId id="F1"/>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>60000</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>382200</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <exerciseDate>
                <value>2003-07-02</value>
            </exerciseDate>
            <expirationDate>
                <value>2010-07-02</value>
            </expirationDate>
            <underlyingSecurity>
                <underlyingSecurityTitle>
                    <value>common stock</value>
                </underlyingSecurityTitle>
                <underlyingSecurityShares>
                    <value>60000</value>
                </underlyingSecurityShares>
            </underlyingSecurity>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>60000</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value></value>
                </natureOfOwnership>
            </ownershipNature>
        </derivativeTransaction>
        <derivativeTransaction>
            <securityTitle>
                <value>Warrants (right to buy)</value>
            </securityTitle>
            <conversionOrExercisePrice>
                <value>7.36</value>
            </conversionOrExercisePrice>
            <transactionDate>
                <value>2006-05-23</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>J</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
                <footnoteId id="F1"/>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>20000</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>105200</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <exerciseDate>
                <value>2003-09-25</value>
            </exerciseDate>
            <expirationDate>
                <value>2010-07-02</value>
            </expirationDate>
            <underlyingSecurity>
                <underlyingSecurityTitle>
                    <value>common stock</value>
                </underlyingSecurityTitle>
                <underlyingSecurityShares>
                    <value>20000</value>
                </underlyingSecurityShares>
            </underlyingSecurity>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>20000</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value></value>
                </natureOfOwnership>
            </ownershipNature>
        </derivativeTransaction>
    </derivativeTable>

    <footnotes>
        <footnote id="F1">The warrants to purchase 80,000 shares of common stock of the Issuer that are the subject of this Form 4 (the &quot;Warrants&quot;) were previously reported as beneficially owned by the Reporting Person indirectly through Jewelcor Management, Inc., a corporation of which the Reporting Person is a controlling shareholder (&quot;Jewelcor&quot;).  This Form 4 reports the change in the form of beneficial ownership of the Warrants which occurred pursuant to the May 23, 2006 acquisition of direct ownership by the Reporting Person of such Warrants from Jewelcor.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/ Seymour Holtzman</signatureName>
        <signatureDate>2006-05-25</signatureDate>
    </ownerSignature>
</ownershipDocument>
</XML>
</TEXT>
</DOCUMENT>
</SEC-DOCUMENT>
-----END PRIVACY-ENHANCED MESSAGE-----