Help with a Stored Procedure

I am having trouble with a stored procedure, seen below. The problem I am having is that this stored proc returns the top 2 users across all offices, not by each office which is what I want. The first part seems to work in that it is finding the average call, but after that I want to return the top 2 highest callers from each office. And what I have so far is that it returns all of them. (I also want to include the offices that have nulls.

CREATE PROC People.spUpdateAndShowAvgCalls 
AS
BEGIN
	BEGIN TRAN
		--find the average time per call for each user
		UPDATE X
		SET X.AverageTimePerCall = Y.AvgTime 
		FROM tblPhoneUsers X inner join (
			SELECT A.UserID, AVG(B.Duration) as AvgTime
			FROM tblPhoneUsers A
			inner join VOIPDatabase.VOIP.[tblPhoneCalls] B on (
				A.PhoneNumber = B.CallingPartyNumber
				OR A.PhoneNumber = B.CalledPartyNumber
			)
			GROUP BY A.UserID
		) Y on X.UserID = Y.UserID; 
		--show the resultset
		SELECT top 2 B.OfficeID, B.OfficeName, A.UserName, A.PhoneNumber, A.AverageTimePerCall
		FROM tblPhoneUsers A
		right outer join tblPhoneOffices B on A.OfficeID = B.OfficeID 
		ORDER BY AverageTimePerCall
	COMMIT TRAN
END

For the sake of clarity and for those who can offer assistance, let me paste the schema code with some sample data (three tables in all)

CREATE SCHEMA [People]
GO
CREATE SCHEMA [VOIP]
GO
CREATE TABLE [People].[tblPhoneUsers](
	[UserID] varchar(50) NOT NULL, --this is the primary key:)
	[OfficeID] varchar(3) NOT NULL,
	[PhoneNumber] varchar(10) NULL,
	[AverageTimePerCall] int NULL --stores how long, on average, the user spends on the phone (incoming and outgoing calls)
)
GO
CREATE TABLE [People].[tblPhoneOffices](
	[OfficeID] varchar(3) NOT NULL,  --this is the primary key:)
	[OfficeName] varchar(300) NOT NULL
	---some more fields here---
)
Go

CREATE TABLE [VOIP].[tblPhoneCalls](
	[CallID] [int] IDENTITY(1,1) NOT NULL,  --this is the primary key:)
	[CallingPartyNumber] [varchar](10) NOT NULL, --number of the person who initiated the call
	[CalledPartyNumber] [varchar](10) NOT NULL, --number of the person who received the call
	[DateTimeConnect] [datetime] NOT NULL, --time that the call was placed
	[Duration] [int] NOT NULL  --duration of the call, in seconds
)
GO 

INSERT INTO [People].[tblPhoneOffices] SELECT '001', 'San Francisco'
INSERT INTO [People].[tblPhoneOffices] SELECT '002', 'Encino'
INSERT INTO [People].[tblPhoneOffices] SELECT '003', 'Calabasas'
INSERT INTO [People].[tblPhoneOffices] SELECT '004', 'Portland'

INSERT INTO [People].[tblPhoneUsers] (UserID, OfficeID, PhoneNumber) SELECT 'abc', '001', '8188188181'
INSERT INTO [People].[tblPhoneUsers] (UserID, OfficeID, PhoneNumber) SELECT 'def', '001', '2122122121'
INSERT INTO [People].[tblPhoneUsers] (UserID, OfficeID, PhoneNumber) SELECT 'ghi', '001', '3133133131'
INSERT INTO [People].[tblPhoneUsers] (UserID, OfficeID, PhoneNumber) SELECT 'jkl', '002', '5455455454'
INSERT INTO [People].[tblPhoneUsers] (UserID, OfficeID, PhoneNumber) SELECT 'mno', '002', '6466466464'
INSERT INTO [People].[tblPhoneUsers] (UserID, OfficeID, PhoneNumber) SELECT 'pqr', '002', '9799799797'

SET IDENTITY_INSERT  [VOIP].[tblPhoneCalls] ON 
INSERT INTO [VOIP].[tblPhoneCalls] (CallID, CallingPartyNumber, CalledPartyNumber, DateTimeConnect, Duration) SELECT 1, '8188188181', '2122122121', '1/1/2011', 100
INSERT INTO [VOIP].[tblPhoneCalls] (CallID, CallingPartyNumber, CalledPartyNumber, DateTimeConnect, Duration) SELECT 2, '2122122121', '4351813854', '1/2/2011', 200
INSERT INTO [VOIP].[tblPhoneCalls] (CallID, CallingPartyNumber, CalledPartyNumber, DateTimeConnect, Duration) SELECT 3, '3133133131', '2122122121', '1/1/2011', 20
INSERT INTO [VOIP].[tblPhoneCalls] (CallID, CallingPartyNumber, CalledPartyNumber, DateTimeConnect, Duration) SELECT 4, '5455455454', '2122122121', '1/1/2011', 100
INSERT INTO [VOIP].[tblPhoneCalls] (CallID, CallingPartyNumber, CalledPartyNumber, DateTimeConnect, Duration) SELECT 5, '5455455454', '9799799797', '1/1/2011', 100
SET IDENTITY_INSERT  [VOIP].[tblPhoneCalls] OFF
GO

What I am trying for is the following result:

Any suggestions would be appreciated.