def __init__(self, action_dim, device):
        super(DQN, self).__init__()
        self.__conv1 = nn.Conv2d(4, 32, kernel_size=8, stride=4, bias=False)
        self.__conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2, bias=False)
        self.__conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1, bias=False)
        self.__fc1_a = nn.Linear(64*7*7, 512)
        self.__fc1_v = nn.Linear(64*7*7, 512)
        self.__fc2_a = nn.Linear(512, action_dim)
        self.__fc2_v = nn.Linear(512, 1)
#         self.__fc3 = nn.Linear(512, 1)
        self.__act_dim = action_dim
        self.__device = device

    def forward(self, x):
        x = x / 255.
        x = F.relu(self.__conv1(x))
        x = F.relu(self.__conv2(x))
        x = F.relu(self.__conv3(x))
        xv = x.view(x.size(0), -1)
        vs = F.relu(self.__fc1_v(xv))
        vs = self.__fc2_v(vs).expand(x.size(0), self.__act_dim)#V
        asa = F.relu(self.__fc1_a(xv))
        asa = self.__fc2_a(asa)#A

#         return vs + asa - torch.mean(asa,-1,True) 
        return vs + asa - asa.mean(1).unsqueeze(1).expand(x.size(0),self.__act_dim)#Q

    def learn(self, memory: ReplayMemory, batch_size: int) -> float:
        """learn trains the value network via TD-learning."""
        state_batch, action_batch, reward_batch, next_batch, done_batch = \
            memory.sample(batch_size)

        values = self.__policy(state_batch.float()).gather(1, action_batch)
        values_next = self.__target(next_batch.float()).max(1).values.detach()
        expected = (self.__gamma * values_next.unsqueeze(1)) * \
            (1. - done_batch) + reward_batch
        loss = F.smooth_l1_loss(values, expected)

        self.__optimizer.zero_grad()
        loss.backward()
        for param in self.__policy.parameters():
            param.grad.data.clamp_(-1, 1)
        self.__optimizer.step()

        return loss.item()

class nbit_flip:

    def __init__(self,stringlength=8,goal=[]):
        self.goalstring = np.random.choice([0,1],stringlength)
        self.startstring = np.random.choice([0,1],stringlength)
        while (self.startstring == self.goalstring).all():
            self.startstring = np.random.choice([0,1],stringlength)
        self.statestring = self.startstring      

    def random_action(self):
        return np.random.randint(0,self.goalstring.shape[0])

    def reset(self):
        self.goalstring = np.random.choice([0,1],self.goalstring.shape[0])
        self.startstring = np.random.choice([0,1],self.goalstring.shape[0])
        while (self.startstring == self.goalstring).all():
            self.startstring = np.random.choice([0,1],self.goalstring.shape[0])
        self.statestring = self.startstring       
        return np.concatenate((self.statestring,self.goalstring))#将状态与目标结合

    def step(self,action):
        self.statestring[action] = np.logical_not(self.statestring[action]).astype(int)
        if (self.statestring == self.goalstring).all():            
            done = True
            reward = 0#成功时奖励为0，否则为-1
        else: 
            done = False
            reward = -1  
        return np.concatenate((self.statestring,self.goalstring)), reward, done#将状态与目标组合输出

bits = 20  # number of bit for bitflipping game

A1 = nn.layer(bits*2,256)
A1.f = nn.f_relu
AOUT = nn.layer(256,bits)
AOUT.f = nn.f_iden

L1 = nn.layer(bits*2,256)
L1.f = nn.f_relu
LOUT = nn.layer(256,bits)
LOUT.f = nn.f_iden

onlineNet = nn.mlp([A1,AOUT]) #if u r numpy_net instead of mlp_framework it's nn.module([...])
for L in onlineNet.Layerlist:
    try:
        L.eta = 1e-3
    except:
        pass
targetNet = nn.mlp([L1,LOUT]) #if u r numpy_net instead of mlp_framework it's nn.module([...])
for L in targetNet.Layerlist:
    try:
        L.eta = 1e-3
    except:
        pass

HER = True ## toggle Hindsight Experience Replay

loss_plot = []
rew_plot = []

ringB = DF.ringbuffer(1e5)

MAX_EPOCHS = 6
MAX_CYCLES = 50
MAX_EPISODES = 16
MAX_STEPS = bits
UPDATE_STEPS = 40
EXPLORATION = 500

BATCHSIZE = 128
GAMMA = 0.98
E_MIN = 0.02


game = nbit_flip(bits)

step_count = 0
success_rate = []

for E in range(MAX_EPOCHS):

    #run cycle
    for c in range(MAX_CYCLES):
        successes = []

        #run episodes
        for e in range(MAX_EPISODES):
            eps_reward = 0
            episode_buffer = []
            current_state = game.reset()

            #run game
            for p in range(MAX_STEPS):
                e = 1. / (((step_count)/EXPLORATION)+1)

                if np.random.uniform(0,1) < max(E_MIN,e):
                    action = game.random_action()
                else:
                    action = np.argmax(onlineNet.infer(current_state[True,:]))

                next_state, reward, done = game.step(action)
                eps_reward += reward
                step_count += 1
                episode_buffer += [(current_state, action, reward, next_state,done)]
                current_state = next_state
                if done or (p+1)==MAX_STEPS: 
                    successes += [done]
                    rew_plot += [eps_reward]
                    break
            #run replay   
            for r in range(p):
                # add each episode step to RPB
                s = np.array(episode_buffer[r][0])
                a = np.array([episode_buffer[r][1]])
                rew = np.array([episode_buffer[r][2]])
                s_ = np.array(episode_buffer[r][3])
                d = np.array([episode_buffer[r][4]])
                ringB.add([s,a,rew,s_,d])

                #run hindsight experience replay
                if HER:
                    # take last state as goal and add (whole) episode to RPB
                    g = np.array(episode_buffer[p-1][3][:bits])#g'
                    s = np.concatenate((s[:bits],g))
                    if (s_[:bits] == g).all(): 
                        rew = np.array([0])
                        d = np.array([True])
                    else:
                        rew = np.array([-1])
                        d = np.array([False])
                    s_ = np.concatenate((s_[:bits],g))
                    ringB.add([s,a,rew,s_,d])

        if (c+1) % 10 == 0: print(np.mean(successes),' cycle success rate')
        success_rate += [np.mean(successes)]

        #run training
        if step_count > BATCHSIZE:
            for u in range(UPDATE_STEPS):
                BATCH = ringB.sample(BATCHSIZE)
                DF.train_bellman(onlineNet, targetNet, BATCH, GAMMA)
                loss_plot += [onlineNet.loss]

            #update target net
            DF.update_target(onlineNet,targetNet)

    print(np.mean(success_rate[-MAX_CYCLES:]),' epoch {}/{} success rate'.format(E+1,MAX_EPOCHS))