001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hdfs.server.blockmanagement;
020
021import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY;
022import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT;
023
024import java.util.Collection;
025import java.util.Random;
026
027import org.apache.commons.logging.Log;
028import org.apache.commons.logging.LogFactory;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hdfs.DFSConfigKeys;
031import org.apache.hadoop.net.NetworkTopology;
032import org.apache.hadoop.net.Node;
033
034/**
035 * Space balanced block placement policy.
036 */
037public class AvailableSpaceBlockPlacementPolicy extends
038    BlockPlacementPolicyDefault {
039  private static final Log LOG = LogFactory
040      .getLog(AvailableSpaceBlockPlacementPolicy.class);
041  private static final Random RAND = new Random();
042  private int balancedPreference =
043      (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
044
045  @Override
046  public void initialize(Configuration conf, FSClusterStats stats,
047      NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) {
048    super.initialize(conf, stats, clusterMap, host2datanodeMap);
049    float balancedPreferencePercent =
050        conf.getFloat(
051          DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY,
052          DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
053
054    LOG.info("Available space block placement policy initialized: "
055        + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
056        + " = " + balancedPreferencePercent);
057
058    if (balancedPreferencePercent > 1.0) {
059      LOG.warn("The value of "
060          + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
061          + " is greater than 1.0 but should be in the range 0.0 - 1.0");
062    }
063    if (balancedPreferencePercent < 0.5) {
064      LOG.warn("The value of "
065          + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
066          + " is less than 0.5 so datanodes with more used percent will"
067          + " receive  more block allocations.");
068    }
069    balancedPreference = (int) (100 * balancedPreferencePercent);
070  }
071
072  @Override
073  protected DatanodeDescriptor chooseDataNode(final String scope,
074      final Collection<Node> excludedNode) {
075    DatanodeDescriptor a =
076        (DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode);
077    DatanodeDescriptor b =
078        (DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode);
079    if (a != null && b != null){
080      int ret = compareDataNode(a, b);
081      if (ret == 0) {
082        return a;
083      } else if (ret < 0) {
084        return (RAND.nextInt(100) < balancedPreference) ? a : b;
085      } else {
086        return (RAND.nextInt(100) < balancedPreference) ? b : a;
087      }
088    } else {
089      return a == null ? b : a;
090    }
091  }
092
093  /**
094   * Compare the two data nodes.
095   */
096  protected int compareDataNode(final DatanodeDescriptor a,
097      final DatanodeDescriptor b) {
098    if (a.equals(b)
099        || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5) {
100      return 0;
101    }
102    return a.getDfsUsedPercent() < b.getDfsUsedPercent() ? -1 : 1;
103  }
104}